GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

openai/openai-python

Public

mirrored from https://github.com/openai/openai-pythonAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
v1.52.0

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openai-python/tests/lib/chat

tests/lib/chat/test_completions_streaming.py

1072lines · modecode

RawDownload
Latest commit unavailable.
1from __future__ import annotations
2
3import os
4from typing import Any, Generic, Callable, Iterator, cast, overload
5from typing_extensions import Literal, TypeVar
6
7import rich
8import httpx
9import pytest
10from respx import MockRouter
11from pydantic import BaseModel
12from inline_snapshot import external, snapshot, outsource
13
14import openai
15from openai import OpenAI, AsyncOpenAI
16from openai._utils import assert_signatures_in_sync
17from openai._compat import model_copy
18from openai.lib.streaming.chat import (
19 ContentDoneEvent,
20 ChatCompletionStream,
21 ChatCompletionStreamEvent,
22 ChatCompletionStreamManager,
23 ParsedChatCompletionSnapshot,
24)
25from openai.lib._parsing._completions import ResponseFormatT
26
27from ._utils import print_obj
28from ...conftest import base_url
29
30_T = TypeVar("_T")
31
32# all the snapshots in this file are auto-generated from the live API
33#
34# you can update them with
35#
36# `OPENAI_LIVE=1 pytest --inline-snapshot=fix`
37
38
39@pytest.mark.respx(base_url=base_url)
40def test_parse_nothing(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
41 listener = _make_stream_snapshot_request(
42 lambda c: c.beta.chat.completions.stream(
43 model="gpt-4o-2024-08-06",
44 messages=[
45 {
46 "role": "user",
47 "content": "What's the weather like in SF?",
48 },
49 ],
50 ),
51 content_snapshot=snapshot(external("e2aad469b71d*.bin")),
52 mock_client=client,
53 respx_mock=respx_mock,
54 )
55
56 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
57 """\
58[
59 ParsedChoice[NoneType](
60 finish_reason='stop',
61 index=0,
62 logprobs=None,
63 message=ParsedChatCompletionMessage[NoneType](
64 audio=None,
65 content="I'm unable to provide real-time weather updates. To get the current weather in San Francisco, I
66recommend checking a reliable weather website or a weather app.",
67 function_call=None,
68 parsed=None,
69 refusal=None,
70 role='assistant',
71 tool_calls=[]
72 )
73 )
74]
75"""
76 )
77 assert print_obj(listener.get_event_by_type("content.done"), monkeypatch) == snapshot(
78 """\
79ContentDoneEvent[NoneType](
80 content="I'm unable to provide real-time weather updates. To get the current weather in San Francisco, I recommend
81checking a reliable weather website or a weather app.",
82 parsed=None,
83 type='content.done'
84)
85"""
86 )
87
88
89@pytest.mark.respx(base_url=base_url)
90def test_parse_pydantic_model(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
91 class Location(BaseModel):
92 city: str
93 temperature: float
94 units: Literal["c", "f"]
95
96 done_snapshots: list[ParsedChatCompletionSnapshot] = []
97
98 def on_event(stream: ChatCompletionStream[Location], event: ChatCompletionStreamEvent[Location]) -> None:
99 if event.type == "content.done":
100 done_snapshots.append(model_copy(stream.current_completion_snapshot, deep=True))
101
102 listener = _make_stream_snapshot_request(
103 lambda c: c.beta.chat.completions.stream(
104 model="gpt-4o-2024-08-06",
105 messages=[
106 {
107 "role": "user",
108 "content": "What's the weather like in SF?",
109 },
110 ],
111 response_format=Location,
112 ),
113 content_snapshot=snapshot(external("7e5ea4d12e7c*.bin")),
114 mock_client=client,
115 respx_mock=respx_mock,
116 on_event=on_event,
117 )
118
119 assert len(done_snapshots) == 1
120 assert isinstance(done_snapshots[0].choices[0].message.parsed, Location)
121
122 for event in reversed(listener.events):
123 if event.type == "content.delta":
124 data = cast(Any, event.parsed)
125 assert isinstance(data["city"], str), data
126 assert isinstance(data["temperature"], (int, float)), data
127 assert isinstance(data["units"], str), data
128 break
129 else:
130 rich.print(listener.events)
131 raise AssertionError("Did not find a `content.delta` event")
132
133 assert print_obj(listener.stream.get_final_completion(), monkeypatch) == snapshot(
134 """\
135ParsedChatCompletion[Location](
136 choices=[
137 ParsedChoice[Location](
138 finish_reason='stop',
139 index=0,
140 logprobs=None,
141 message=ParsedChatCompletionMessage[Location](
142 audio=None,
143 content='{"city":"San Francisco","temperature":61,"units":"f"}',
144 function_call=None,
145 parsed=Location(city='San Francisco', temperature=61.0, units='f'),
146 refusal=None,
147 role='assistant',
148 tool_calls=[]
149 )
150 )
151 ],
152 created=1727346169,
153 id='chatcmpl-ABfw1e5abtU8OwGr15vOreYVb2MiF',
154 model='gpt-4o-2024-08-06',
155 object='chat.completion',
156 service_tier=None,
157 system_fingerprint='fp_5050236cbd',
158 usage=CompletionUsage(
159 completion_tokens=14,
160 completion_tokens_details=CompletionTokensDetails(audio_tokens=None, reasoning_tokens=0),
161 prompt_tokens=79,
162 prompt_tokens_details=None,
163 total_tokens=93
164 )
165)
166"""
167 )
168 assert print_obj(listener.get_event_by_type("content.done"), monkeypatch) == snapshot(
169 """\
170ContentDoneEvent[Location](
171 content='{"city":"San Francisco","temperature":61,"units":"f"}',
172 parsed=Location(city='San Francisco', temperature=61.0, units='f'),
173 type='content.done'
174)
175"""
176 )
177
178
179@pytest.mark.respx(base_url=base_url)
180def test_parse_pydantic_model_multiple_choices(
181 client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch
182) -> None:
183 class Location(BaseModel):
184 city: str
185 temperature: float
186 units: Literal["c", "f"]
187
188 listener = _make_stream_snapshot_request(
189 lambda c: c.beta.chat.completions.stream(
190 model="gpt-4o-2024-08-06",
191 messages=[
192 {
193 "role": "user",
194 "content": "What's the weather like in SF?",
195 },
196 ],
197 n=3,
198 response_format=Location,
199 ),
200 content_snapshot=snapshot(external("a491adda08c3*.bin")),
201 mock_client=client,
202 respx_mock=respx_mock,
203 )
204
205 assert [e.type for e in listener.events] == snapshot(
206 [
207 "chunk",
208 "content.delta",
209 "chunk",
210 "content.delta",
211 "chunk",
212 "content.delta",
213 "chunk",
214 "content.delta",
215 "chunk",
216 "content.delta",
217 "chunk",
218 "content.delta",
219 "chunk",
220 "content.delta",
221 "chunk",
222 "content.delta",
223 "chunk",
224 "content.delta",
225 "chunk",
226 "content.delta",
227 "chunk",
228 "content.delta",
229 "chunk",
230 "content.delta",
231 "chunk",
232 "content.delta",
233 "chunk",
234 "content.delta",
235 "chunk",
236 "content.delta",
237 "chunk",
238 "content.delta",
239 "chunk",
240 "content.delta",
241 "chunk",
242 "content.delta",
243 "chunk",
244 "content.delta",
245 "chunk",
246 "content.delta",
247 "chunk",
248 "content.delta",
249 "chunk",
250 "content.delta",
251 "chunk",
252 "content.delta",
253 "chunk",
254 "content.delta",
255 "chunk",
256 "content.delta",
257 "chunk",
258 "content.delta",
259 "chunk",
260 "content.delta",
261 "chunk",
262 "content.delta",
263 "chunk",
264 "content.delta",
265 "chunk",
266 "content.delta",
267 "chunk",
268 "content.delta",
269 "chunk",
270 "content.delta",
271 "chunk",
272 "content.delta",
273 "chunk",
274 "content.delta",
275 "chunk",
276 "content.delta",
277 "chunk",
278 "content.delta",
279 "chunk",
280 "content.delta",
281 "chunk",
282 "content.delta",
283 "chunk",
284 "content.delta",
285 "chunk",
286 "content.delta",
287 "chunk",
288 "content.delta",
289 "chunk",
290 "content.delta",
291 "chunk",
292 "content.delta",
293 "chunk",
294 "content.delta",
295 "chunk",
296 "content.delta",
297 "chunk",
298 "content.done",
299 "chunk",
300 "content.done",
301 "chunk",
302 "content.done",
303 "chunk",
304 ]
305 )
306 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
307 """\
308[
309 ParsedChoice[Location](
310 finish_reason='stop',
311 index=0,
312 logprobs=None,
313 message=ParsedChatCompletionMessage[Location](
314 audio=None,
315 content='{"city":"San Francisco","temperature":65,"units":"f"}',
316 function_call=None,
317 parsed=Location(city='San Francisco', temperature=65.0, units='f'),
318 refusal=None,
319 role='assistant',
320 tool_calls=[]
321 )
322 ),
323 ParsedChoice[Location](
324 finish_reason='stop',
325 index=1,
326 logprobs=None,
327 message=ParsedChatCompletionMessage[Location](
328 audio=None,
329 content='{"city":"San Francisco","temperature":61,"units":"f"}',
330 function_call=None,
331 parsed=Location(city='San Francisco', temperature=61.0, units='f'),
332 refusal=None,
333 role='assistant',
334 tool_calls=[]
335 )
336 ),
337 ParsedChoice[Location](
338 finish_reason='stop',
339 index=2,
340 logprobs=None,
341 message=ParsedChatCompletionMessage[Location](
342 audio=None,
343 content='{"city":"San Francisco","temperature":59,"units":"f"}',
344 function_call=None,
345 parsed=Location(city='San Francisco', temperature=59.0, units='f'),
346 refusal=None,
347 role='assistant',
348 tool_calls=[]
349 )
350 )
351]
352"""
353 )
354
355
356@pytest.mark.respx(base_url=base_url)
357def test_parse_max_tokens_reached(client: OpenAI, respx_mock: MockRouter) -> None:
358 class Location(BaseModel):
359 city: str
360 temperature: float
361 units: Literal["c", "f"]
362
363 with pytest.raises(openai.LengthFinishReasonError):
364 _make_stream_snapshot_request(
365 lambda c: c.beta.chat.completions.stream(
366 model="gpt-4o-2024-08-06",
367 messages=[
368 {
369 "role": "user",
370 "content": "What's the weather like in SF?",
371 },
372 ],
373 max_tokens=1,
374 response_format=Location,
375 ),
376 content_snapshot=snapshot(external("4cc50a6135d2*.bin")),
377 mock_client=client,
378 respx_mock=respx_mock,
379 )
380
381
382@pytest.mark.respx(base_url=base_url)
383def test_parse_pydantic_model_refusal(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
384 class Location(BaseModel):
385 city: str
386 temperature: float
387 units: Literal["c", "f"]
388
389 listener = _make_stream_snapshot_request(
390 lambda c: c.beta.chat.completions.stream(
391 model="gpt-4o-2024-08-06",
392 messages=[
393 {
394 "role": "user",
395 "content": "How do I make anthrax?",
396 },
397 ],
398 response_format=Location,
399 ),
400 content_snapshot=snapshot(external("173417d55340*.bin")),
401 mock_client=client,
402 respx_mock=respx_mock,
403 )
404
405 assert print_obj(listener.get_event_by_type("refusal.done"), monkeypatch) == snapshot("""\
406RefusalDoneEvent(refusal="I'm sorry, I can't assist with that request.", type='refusal.done')
407""")
408
409 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
410 """\
411[
412 ParsedChoice[Location](
413 finish_reason='stop',
414 index=0,
415 logprobs=None,
416 message=ParsedChatCompletionMessage[Location](
417 audio=None,
418 content=None,
419 function_call=None,
420 parsed=None,
421 refusal="I'm sorry, I can't assist with that request.",
422 role='assistant',
423 tool_calls=[]
424 )
425 )
426]
427"""
428 )
429
430
431@pytest.mark.respx(base_url=base_url)
432def test_content_logprobs_events(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
433 listener = _make_stream_snapshot_request(
434 lambda c: c.beta.chat.completions.stream(
435 model="gpt-4o-2024-08-06",
436 messages=[
437 {
438 "role": "user",
439 "content": "Say foo",
440 },
441 ],
442 logprobs=True,
443 ),
444 content_snapshot=snapshot(external("83b060bae42e*.bin")),
445 mock_client=client,
446 respx_mock=respx_mock,
447 )
448
449 assert print_obj([e for e in listener.events if e.type.startswith("logprobs")], monkeypatch) == snapshot("""\
450[
451 LogprobsContentDeltaEvent(
452 content=[
453 ChatCompletionTokenLogprob(bytes=[70, 111, 111], logprob=-0.0025094282, token='Foo', top_logprobs=[])
454 ],
455 snapshot=[
456 ChatCompletionTokenLogprob(bytes=[70, 111, 111], logprob=-0.0025094282, token='Foo', top_logprobs=[])
457 ],
458 type='logprobs.content.delta'
459 ),
460 LogprobsContentDeltaEvent(
461 content=[ChatCompletionTokenLogprob(bytes=[33], logprob=-0.26638845, token='!', top_logprobs=[])],
462 snapshot=[
463 ChatCompletionTokenLogprob(bytes=[70, 111, 111], logprob=-0.0025094282, token='Foo', top_logprobs=[]),
464 ChatCompletionTokenLogprob(bytes=[33], logprob=-0.26638845, token='!', top_logprobs=[])
465 ],
466 type='logprobs.content.delta'
467 ),
468 LogprobsContentDoneEvent(
469 content=[
470 ChatCompletionTokenLogprob(bytes=[70, 111, 111], logprob=-0.0025094282, token='Foo', top_logprobs=[]),
471 ChatCompletionTokenLogprob(bytes=[33], logprob=-0.26638845, token='!', top_logprobs=[])
472 ],
473 type='logprobs.content.done'
474 )
475]
476""")
477
478 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot("""\
479[
480 ParsedChoice[NoneType](
481 finish_reason='stop',
482 index=0,
483 logprobs=ChoiceLogprobs(
484 content=[
485 ChatCompletionTokenLogprob(bytes=[70, 111, 111], logprob=-0.0025094282, token='Foo', top_logprobs=[]),
486 ChatCompletionTokenLogprob(bytes=[33], logprob=-0.26638845, token='!', top_logprobs=[])
487 ],
488 refusal=None
489 ),
490 message=ParsedChatCompletionMessage[NoneType](
491 audio=None,
492 content='Foo!',
493 function_call=None,
494 parsed=None,
495 refusal=None,
496 role='assistant',
497 tool_calls=[]
498 )
499 )
500]
501""")
502
503
504@pytest.mark.respx(base_url=base_url)
505def test_refusal_logprobs_events(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
506 class Location(BaseModel):
507 city: str
508 temperature: float
509 units: Literal["c", "f"]
510
511 listener = _make_stream_snapshot_request(
512 lambda c: c.beta.chat.completions.stream(
513 model="gpt-4o-2024-08-06",
514 messages=[
515 {
516 "role": "user",
517 "content": "How do I make anthrax?",
518 },
519 ],
520 logprobs=True,
521 response_format=Location,
522 ),
523 content_snapshot=snapshot(external("569c877e6942*.bin")),
524 mock_client=client,
525 respx_mock=respx_mock,
526 )
527
528 assert print_obj([e.type for e in listener.events if e.type.startswith("logprobs")], monkeypatch) == snapshot("""\
529[
530 'logprobs.refusal.delta',
531 'logprobs.refusal.delta',
532 'logprobs.refusal.delta',
533 'logprobs.refusal.delta',
534 'logprobs.refusal.delta',
535 'logprobs.refusal.delta',
536 'logprobs.refusal.delta',
537 'logprobs.refusal.delta',
538 'logprobs.refusal.delta',
539 'logprobs.refusal.delta',
540 'logprobs.refusal.delta',
541 'logprobs.refusal.done'
542]
543""")
544
545 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot("""\
546[
547 ParsedChoice[Location](
548 finish_reason='stop',
549 index=0,
550 logprobs=ChoiceLogprobs(
551 content=None,
552 refusal=[
553 ChatCompletionTokenLogprob(bytes=[73, 39, 109], logprob=-0.0012038043, token="I'm", top_logprobs=[]),
554 ChatCompletionTokenLogprob(
555 bytes=[32, 118, 101, 114, 121],
556 logprob=-0.8438816,
557 token=' very',
558 top_logprobs=[]
559 ),
560 ChatCompletionTokenLogprob(
561 bytes=[32, 115, 111, 114, 114, 121],
562 logprob=-3.4121115e-06,
563 token=' sorry',
564 top_logprobs=[]
565 ),
566 ChatCompletionTokenLogprob(bytes=[44], logprob=-3.3809047e-05, token=',', top_logprobs=[]),
567 ChatCompletionTokenLogprob(
568 bytes=[32, 98, 117, 116],
569 logprob=-0.038048144,
570 token=' but',
571 top_logprobs=[]
572 ),
573 ChatCompletionTokenLogprob(bytes=[32, 73], logprob=-0.0016109125, token=' I', top_logprobs=[]),
574 ChatCompletionTokenLogprob(
575 bytes=[32, 99, 97, 110, 39, 116],
576 logprob=-0.0073532974,
577 token=" can't",
578 top_logprobs=[]
579 ),
580 ChatCompletionTokenLogprob(
581 bytes=[32, 97, 115, 115, 105, 115, 116],
582 logprob=-0.0020837625,
583 token=' assist',
584 top_logprobs=[]
585 ),
586 ChatCompletionTokenLogprob(
587 bytes=[32, 119, 105, 116, 104],
588 logprob=-0.00318354,
589 token=' with',
590 top_logprobs=[]
591 ),
592 ChatCompletionTokenLogprob(
593 bytes=[32, 116, 104, 97, 116],
594 logprob=-0.0017186158,
595 token=' that',
596 top_logprobs=[]
597 ),
598 ChatCompletionTokenLogprob(bytes=[46], logprob=-0.57687104, token='.', top_logprobs=[])
599 ]
600 ),
601 message=ParsedChatCompletionMessage[Location](
602 audio=None,
603 content=None,
604 function_call=None,
605 parsed=None,
606 refusal="I'm very sorry, but I can't assist with that.",
607 role='assistant',
608 tool_calls=[]
609 )
610 )
611]
612""")
613
614
615@pytest.mark.respx(base_url=base_url)
616def test_parse_pydantic_tool(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
617 class GetWeatherArgs(BaseModel):
618 city: str
619 country: str
620 units: Literal["c", "f"] = "c"
621
622 listener = _make_stream_snapshot_request(
623 lambda c: c.beta.chat.completions.stream(
624 model="gpt-4o-2024-08-06",
625 messages=[
626 {
627 "role": "user",
628 "content": "What's the weather like in Edinburgh?",
629 },
630 ],
631 tools=[
632 openai.pydantic_function_tool(GetWeatherArgs),
633 ],
634 ),
635 content_snapshot=snapshot(external("c6aa7e397b71*.bin")),
636 mock_client=client,
637 respx_mock=respx_mock,
638 )
639
640 assert print_obj(listener.stream.current_completion_snapshot.choices, monkeypatch) == snapshot(
641 """\
642[
643 ParsedChoice[object](
644 finish_reason='tool_calls',
645 index=0,
646 logprobs=None,
647 message=ParsedChatCompletionMessage[object](
648 audio=None,
649 content=None,
650 function_call=None,
651 parsed=None,
652 refusal=None,
653 role='assistant',
654 tool_calls=[
655 ParsedFunctionToolCall(
656 function=ParsedFunction(
657 arguments='{"city":"Edinburgh","country":"UK","units":"c"}',
658 name='GetWeatherArgs',
659 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='UK', units='c')
660 ),
661 id='call_c91SqDXlYFuETYv8mUHzz6pp',
662 index=0,
663 type='function'
664 )
665 ]
666 )
667 )
668]
669"""
670 )
671
672 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
673 """\
674[
675 ParsedChoice[NoneType](
676 finish_reason='tool_calls',
677 index=0,
678 logprobs=None,
679 message=ParsedChatCompletionMessage[NoneType](
680 audio=None,
681 content=None,
682 function_call=None,
683 parsed=None,
684 refusal=None,
685 role='assistant',
686 tool_calls=[
687 ParsedFunctionToolCall(
688 function=ParsedFunction(
689 arguments='{"city":"Edinburgh","country":"UK","units":"c"}',
690 name='GetWeatherArgs',
691 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='UK', units='c')
692 ),
693 id='call_c91SqDXlYFuETYv8mUHzz6pp',
694 index=0,
695 type='function'
696 )
697 ]
698 )
699 )
700]
701"""
702 )
703
704
705@pytest.mark.respx(base_url=base_url)
706def test_parse_multiple_pydantic_tools(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
707 class GetWeatherArgs(BaseModel):
708 """Get the temperature for the given country/city combo"""
709
710 city: str
711 country: str
712 units: Literal["c", "f"] = "c"
713
714 class GetStockPrice(BaseModel):
715 ticker: str
716 exchange: str
717
718 listener = _make_stream_snapshot_request(
719 lambda c: c.beta.chat.completions.stream(
720 model="gpt-4o-2024-08-06",
721 messages=[
722 {
723 "role": "user",
724 "content": "What's the weather like in Edinburgh?",
725 },
726 {
727 "role": "user",
728 "content": "What's the price of AAPL?",
729 },
730 ],
731 tools=[
732 openai.pydantic_function_tool(GetWeatherArgs),
733 openai.pydantic_function_tool(
734 GetStockPrice, name="get_stock_price", description="Fetch the latest price for a given ticker"
735 ),
736 ],
737 ),
738 content_snapshot=snapshot(external("f82268f2fefd*.bin")),
739 mock_client=client,
740 respx_mock=respx_mock,
741 )
742
743 assert print_obj(listener.stream.current_completion_snapshot.choices, monkeypatch) == snapshot(
744 """\
745[
746 ParsedChoice[object](
747 finish_reason='tool_calls',
748 index=0,
749 logprobs=None,
750 message=ParsedChatCompletionMessage[object](
751 audio=None,
752 content=None,
753 function_call=None,
754 parsed=None,
755 refusal=None,
756 role='assistant',
757 tool_calls=[
758 ParsedFunctionToolCall(
759 function=ParsedFunction(
760 arguments='{"city": "Edinburgh", "country": "GB", "units": "c"}',
761 name='GetWeatherArgs',
762 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='GB', units='c')
763 ),
764 id='call_JMW1whyEaYG438VE1OIflxA2',
765 index=0,
766 type='function'
767 ),
768 ParsedFunctionToolCall(
769 function=ParsedFunction(
770 arguments='{"ticker": "AAPL", "exchange": "NASDAQ"}',
771 name='get_stock_price',
772 parsed_arguments=GetStockPrice(exchange='NASDAQ', ticker='AAPL')
773 ),
774 id='call_DNYTawLBoN8fj3KN6qU9N1Ou',
775 index=1,
776 type='function'
777 )
778 ]
779 )
780 )
781]
782"""
783 )
784 completion = listener.stream.get_final_completion()
785 assert print_obj(completion.choices[0].message.tool_calls, monkeypatch) == snapshot(
786 """\
787[
788 ParsedFunctionToolCall(
789 function=ParsedFunction(
790 arguments='{"city": "Edinburgh", "country": "GB", "units": "c"}',
791 name='GetWeatherArgs',
792 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='GB', units='c')
793 ),
794 id='call_JMW1whyEaYG438VE1OIflxA2',
795 index=0,
796 type='function'
797 ),
798 ParsedFunctionToolCall(
799 function=ParsedFunction(
800 arguments='{"ticker": "AAPL", "exchange": "NASDAQ"}',
801 name='get_stock_price',
802 parsed_arguments=GetStockPrice(exchange='NASDAQ', ticker='AAPL')
803 ),
804 id='call_DNYTawLBoN8fj3KN6qU9N1Ou',
805 index=1,
806 type='function'
807 )
808]
809"""
810 )
811
812
813@pytest.mark.respx(base_url=base_url)
814def test_parse_strict_tools(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
815 listener = _make_stream_snapshot_request(
816 lambda c: c.beta.chat.completions.stream(
817 model="gpt-4o-2024-08-06",
818 messages=[
819 {
820 "role": "user",
821 "content": "What's the weather like in SF?",
822 },
823 ],
824 tools=[
825 {
826 "type": "function",
827 "function": {
828 "name": "get_weather",
829 "parameters": {
830 "type": "object",
831 "properties": {
832 "city": {"type": "string"},
833 "state": {"type": "string"},
834 },
835 "required": [
836 "city",
837 "state",
838 ],
839 "additionalProperties": False,
840 },
841 "strict": True,
842 },
843 }
844 ],
845 ),
846 content_snapshot=snapshot(external("a247c49c5fcd*.bin")),
847 mock_client=client,
848 respx_mock=respx_mock,
849 )
850
851 assert print_obj(listener.stream.current_completion_snapshot.choices, monkeypatch) == snapshot(
852 """\
853[
854 ParsedChoice[object](
855 finish_reason='tool_calls',
856 index=0,
857 logprobs=None,
858 message=ParsedChatCompletionMessage[object](
859 audio=None,
860 content=None,
861 function_call=None,
862 parsed=None,
863 refusal=None,
864 role='assistant',
865 tool_calls=[
866 ParsedFunctionToolCall(
867 function=ParsedFunction(
868 arguments='{"city":"San Francisco","state":"CA"}',
869 name='get_weather',
870 parsed_arguments={'city': 'San Francisco', 'state': 'CA'}
871 ),
872 id='call_CTf1nWJLqSeRgDqaCG27xZ74',
873 index=0,
874 type='function'
875 )
876 ]
877 )
878 )
879]
880"""
881 )
882
883
884@pytest.mark.respx(base_url=base_url)
885def test_non_pydantic_response_format(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
886 listener = _make_stream_snapshot_request(
887 lambda c: c.beta.chat.completions.stream(
888 model="gpt-4o-2024-08-06",
889 messages=[
890 {
891 "role": "user",
892 "content": "What's the weather like in SF? Give me any JSON back",
893 },
894 ],
895 response_format={"type": "json_object"},
896 ),
897 content_snapshot=snapshot(external("d61558011839*.bin")),
898 mock_client=client,
899 respx_mock=respx_mock,
900 )
901
902 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
903 """\
904[
905 ParsedChoice[NoneType](
906 finish_reason='stop',
907 index=0,
908 logprobs=None,
909 message=ParsedChatCompletionMessage[NoneType](
910 audio=None,
911 content='\\n {\\n "location": "San Francisco, CA",\\n "weather": {\\n "temperature": "18°C",\\n
912"condition": "Partly Cloudy",\\n "humidity": "72%",\\n "windSpeed": "15 km/h",\\n "windDirection": "NW"\\n
913},\\n "forecast": [\\n {\\n "day": "Monday",\\n "high": "20°C",\\n "low": "14°C",\\n
914"condition": "Sunny"\\n },\\n {\\n "day": "Tuesday",\\n "high": "19°C",\\n "low": "15°C",\\n
915"condition": "Mostly Cloudy"\\n },\\n {\\n "day": "Wednesday",\\n "high": "18°C",\\n "low":
916"14°C",\\n "condition": "Cloudy"\\n }\\n ]\\n }\\n',
917 function_call=None,
918 parsed=None,
919 refusal=None,
920 role='assistant',
921 tool_calls=[]
922 )
923 )
924]
925"""
926 )
927
928
929@pytest.mark.respx(base_url=base_url)
930def test_allows_non_strict_tools_but_no_parsing(
931 client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch
932) -> None:
933 listener = _make_stream_snapshot_request(
934 lambda c: c.beta.chat.completions.stream(
935 model="gpt-4o-2024-08-06",
936 messages=[{"role": "user", "content": "what's the weather in NYC?"}],
937 tools=[
938 {
939 "type": "function",
940 "function": {
941 "name": "get_weather",
942 "parameters": {"type": "object", "properties": {"city": {"type": "string"}}},
943 },
944 }
945 ],
946 ),
947 content_snapshot=snapshot(external("2018feb66ae1*.bin")),
948 mock_client=client,
949 respx_mock=respx_mock,
950 )
951
952 assert print_obj(listener.get_event_by_type("tool_calls.function.arguments.done"), monkeypatch) == snapshot("""\
953FunctionToolCallArgumentsDoneEvent(
954 arguments='{"city":"New York City"}',
955 index=0,
956 name='get_weather',
957 parsed_arguments=None,
958 type='tool_calls.function.arguments.done'
959)
960""")
961
962 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
963 """\
964[
965 ParsedChoice[NoneType](
966 finish_reason='tool_calls',
967 index=0,
968 logprobs=None,
969 message=ParsedChatCompletionMessage[NoneType](
970 audio=None,
971 content=None,
972 function_call=None,
973 parsed=None,
974 refusal=None,
975 role='assistant',
976 tool_calls=[
977 ParsedFunctionToolCall(
978 function=ParsedFunction(
979 arguments='{"city":"New York City"}',
980 name='get_weather',
981 parsed_arguments=None
982 ),
983 id='call_4XzlGBLtUe9dy3GVNV4jhq7h',
984 index=0,
985 type='function'
986 )
987 ]
988 )
989 )
990]
991"""
992 )
993
994
995@pytest.mark.parametrize("sync", [True, False], ids=["sync", "async"])
996def test_stream_method_in_sync(sync: bool, client: OpenAI, async_client: AsyncOpenAI) -> None:
997 checking_client: OpenAI | AsyncOpenAI = client if sync else async_client
998
999 assert_signatures_in_sync(
1000 checking_client.chat.completions.create,
1001 checking_client.beta.chat.completions.stream,
1002 exclude_params={"response_format", "stream"},
1003 )
1004
1005
1006class StreamListener(Generic[ResponseFormatT]):
1007 def __init__(self, stream: ChatCompletionStream[ResponseFormatT]) -> None:
1008 self.stream = stream
1009 self.events: list[ChatCompletionStreamEvent[ResponseFormatT]] = []
1010
1011 def __iter__(self) -> Iterator[ChatCompletionStreamEvent[ResponseFormatT]]:
1012 for event in self.stream:
1013 self.events.append(event)
1014 yield event
1015
1016 @overload
1017 def get_event_by_type(self, event_type: Literal["content.done"]) -> ContentDoneEvent[ResponseFormatT] | None: ...
1018
1019 @overload
1020 def get_event_by_type(self, event_type: str) -> ChatCompletionStreamEvent[ResponseFormatT] | None: ...
1021
1022 def get_event_by_type(self, event_type: str) -> ChatCompletionStreamEvent[ResponseFormatT] | None:
1023 return next((e for e in self.events if e.type == event_type), None)
1024
1025
1026def _make_stream_snapshot_request(
1027 func: Callable[[OpenAI], ChatCompletionStreamManager[ResponseFormatT]],
1028 *,
1029 content_snapshot: Any,
1030 respx_mock: MockRouter,
1031 mock_client: OpenAI,
1032 on_event: Callable[[ChatCompletionStream[ResponseFormatT], ChatCompletionStreamEvent[ResponseFormatT]], Any]
1033 | None = None,
1034) -> StreamListener[ResponseFormatT]:
1035 live = os.environ.get("OPENAI_LIVE") == "1"
1036 if live:
1037
1038 def _on_response(response: httpx.Response) -> None:
1039 # update the content snapshot
1040 assert outsource(response.read()) == content_snapshot
1041
1042 respx_mock.stop()
1043
1044 client = OpenAI(
1045 http_client=httpx.Client(
1046 event_hooks={
1047 "response": [_on_response],
1048 }
1049 )
1050 )
1051 else:
1052 respx_mock.post("/chat/completions").mock(
1053 return_value=httpx.Response(
1054 200,
1055 content=content_snapshot._old_value._load_value(),
1056 headers={"content-type": "text/event-stream"},
1057 )
1058 )
1059
1060 client = mock_client
1061
1062 with func(client) as stream:
1063 listener = StreamListener(stream)
1064
1065 for event in listener:
1066 if on_event:
1067 on_event(stream, event)
1068
1069 if live:
1070 client.close()
1071
1072 return listener
1073