GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

openai/openai-python

Public

mirrored from https://github.com/openai/openai-pythonAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
v1.55.3

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openai-python/tests/lib/chat

tests/lib/chat/test_completions_streaming.py

1077lines · modecode

RawDownload
Latest commit unavailable.
1from __future__ import annotations
2
3import os
4from typing import Any, Generic, Callable, Iterator, cast, overload
5from typing_extensions import Literal, TypeVar
6
7import rich
8import httpx
9import pytest
10from respx import MockRouter
11from pydantic import BaseModel
12from inline_snapshot import external, snapshot, outsource
13
14import openai
15from openai import OpenAI, AsyncOpenAI
16from openai._utils import assert_signatures_in_sync
17from openai._compat import model_copy
18from openai.lib.streaming.chat import (
19 ContentDoneEvent,
20 ChatCompletionStream,
21 ChatCompletionStreamEvent,
22 ChatCompletionStreamManager,
23 ParsedChatCompletionSnapshot,
24)
25from openai.lib._parsing._completions import ResponseFormatT
26
27from ._utils import print_obj
28from ...conftest import base_url
29
30_T = TypeVar("_T")
31
32# all the snapshots in this file are auto-generated from the live API
33#
34# you can update them with
35#
36# `OPENAI_LIVE=1 pytest --inline-snapshot=fix`
37
38
39@pytest.mark.respx(base_url=base_url)
40def test_parse_nothing(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
41 listener = _make_stream_snapshot_request(
42 lambda c: c.beta.chat.completions.stream(
43 model="gpt-4o-2024-08-06",
44 messages=[
45 {
46 "role": "user",
47 "content": "What's the weather like in SF?",
48 },
49 ],
50 ),
51 content_snapshot=snapshot(external("e2aad469b71d*.bin")),
52 mock_client=client,
53 respx_mock=respx_mock,
54 )
55
56 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
57 """\
58[
59 ParsedChoice[NoneType](
60 finish_reason='stop',
61 index=0,
62 logprobs=None,
63 message=ParsedChatCompletionMessage[NoneType](
64 audio=None,
65 content="I'm unable to provide real-time weather updates. To get the current weather in San Francisco, I
66recommend checking a reliable weather website or a weather app.",
67 function_call=None,
68 parsed=None,
69 refusal=None,
70 role='assistant',
71 tool_calls=[]
72 )
73 )
74]
75"""
76 )
77 assert print_obj(listener.get_event_by_type("content.done"), monkeypatch) == snapshot(
78 """\
79ContentDoneEvent[NoneType](
80 content="I'm unable to provide real-time weather updates. To get the current weather in San Francisco, I recommend
81checking a reliable weather website or a weather app.",
82 parsed=None,
83 type='content.done'
84)
85"""
86 )
87
88
89@pytest.mark.respx(base_url=base_url)
90def test_parse_pydantic_model(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
91 class Location(BaseModel):
92 city: str
93 temperature: float
94 units: Literal["c", "f"]
95
96 done_snapshots: list[ParsedChatCompletionSnapshot] = []
97
98 def on_event(stream: ChatCompletionStream[Location], event: ChatCompletionStreamEvent[Location]) -> None:
99 if event.type == "content.done":
100 done_snapshots.append(model_copy(stream.current_completion_snapshot, deep=True))
101
102 listener = _make_stream_snapshot_request(
103 lambda c: c.beta.chat.completions.stream(
104 model="gpt-4o-2024-08-06",
105 messages=[
106 {
107 "role": "user",
108 "content": "What's the weather like in SF?",
109 },
110 ],
111 response_format=Location,
112 ),
113 content_snapshot=snapshot(external("7e5ea4d12e7c*.bin")),
114 mock_client=client,
115 respx_mock=respx_mock,
116 on_event=on_event,
117 )
118
119 assert len(done_snapshots) == 1
120 assert isinstance(done_snapshots[0].choices[0].message.parsed, Location)
121
122 for event in reversed(listener.events):
123 if event.type == "content.delta":
124 data = cast(Any, event.parsed)
125 assert isinstance(data["city"], str), data
126 assert isinstance(data["temperature"], (int, float)), data
127 assert isinstance(data["units"], str), data
128 break
129 else:
130 rich.print(listener.events)
131 raise AssertionError("Did not find a `content.delta` event")
132
133 assert print_obj(listener.stream.get_final_completion(), monkeypatch) == snapshot(
134 """\
135ParsedChatCompletion[Location](
136 choices=[
137 ParsedChoice[Location](
138 finish_reason='stop',
139 index=0,
140 logprobs=None,
141 message=ParsedChatCompletionMessage[Location](
142 audio=None,
143 content='{"city":"San Francisco","temperature":61,"units":"f"}',
144 function_call=None,
145 parsed=Location(city='San Francisco', temperature=61.0, units='f'),
146 refusal=None,
147 role='assistant',
148 tool_calls=[]
149 )
150 )
151 ],
152 created=1727346169,
153 id='chatcmpl-ABfw1e5abtU8OwGr15vOreYVb2MiF',
154 model='gpt-4o-2024-08-06',
155 object='chat.completion',
156 service_tier=None,
157 system_fingerprint='fp_5050236cbd',
158 usage=CompletionUsage(
159 completion_tokens=14,
160 completion_tokens_details=CompletionTokensDetails(
161 accepted_prediction_tokens=None,
162 audio_tokens=None,
163 reasoning_tokens=0,
164 rejected_prediction_tokens=None
165 ),
166 prompt_tokens=79,
167 prompt_tokens_details=None,
168 total_tokens=93
169 )
170)
171"""
172 )
173 assert print_obj(listener.get_event_by_type("content.done"), monkeypatch) == snapshot(
174 """\
175ContentDoneEvent[Location](
176 content='{"city":"San Francisco","temperature":61,"units":"f"}',
177 parsed=Location(city='San Francisco', temperature=61.0, units='f'),
178 type='content.done'
179)
180"""
181 )
182
183
184@pytest.mark.respx(base_url=base_url)
185def test_parse_pydantic_model_multiple_choices(
186 client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch
187) -> None:
188 class Location(BaseModel):
189 city: str
190 temperature: float
191 units: Literal["c", "f"]
192
193 listener = _make_stream_snapshot_request(
194 lambda c: c.beta.chat.completions.stream(
195 model="gpt-4o-2024-08-06",
196 messages=[
197 {
198 "role": "user",
199 "content": "What's the weather like in SF?",
200 },
201 ],
202 n=3,
203 response_format=Location,
204 ),
205 content_snapshot=snapshot(external("a491adda08c3*.bin")),
206 mock_client=client,
207 respx_mock=respx_mock,
208 )
209
210 assert [e.type for e in listener.events] == snapshot(
211 [
212 "chunk",
213 "content.delta",
214 "chunk",
215 "content.delta",
216 "chunk",
217 "content.delta",
218 "chunk",
219 "content.delta",
220 "chunk",
221 "content.delta",
222 "chunk",
223 "content.delta",
224 "chunk",
225 "content.delta",
226 "chunk",
227 "content.delta",
228 "chunk",
229 "content.delta",
230 "chunk",
231 "content.delta",
232 "chunk",
233 "content.delta",
234 "chunk",
235 "content.delta",
236 "chunk",
237 "content.delta",
238 "chunk",
239 "content.delta",
240 "chunk",
241 "content.delta",
242 "chunk",
243 "content.delta",
244 "chunk",
245 "content.delta",
246 "chunk",
247 "content.delta",
248 "chunk",
249 "content.delta",
250 "chunk",
251 "content.delta",
252 "chunk",
253 "content.delta",
254 "chunk",
255 "content.delta",
256 "chunk",
257 "content.delta",
258 "chunk",
259 "content.delta",
260 "chunk",
261 "content.delta",
262 "chunk",
263 "content.delta",
264 "chunk",
265 "content.delta",
266 "chunk",
267 "content.delta",
268 "chunk",
269 "content.delta",
270 "chunk",
271 "content.delta",
272 "chunk",
273 "content.delta",
274 "chunk",
275 "content.delta",
276 "chunk",
277 "content.delta",
278 "chunk",
279 "content.delta",
280 "chunk",
281 "content.delta",
282 "chunk",
283 "content.delta",
284 "chunk",
285 "content.delta",
286 "chunk",
287 "content.delta",
288 "chunk",
289 "content.delta",
290 "chunk",
291 "content.delta",
292 "chunk",
293 "content.delta",
294 "chunk",
295 "content.delta",
296 "chunk",
297 "content.delta",
298 "chunk",
299 "content.delta",
300 "chunk",
301 "content.delta",
302 "chunk",
303 "content.done",
304 "chunk",
305 "content.done",
306 "chunk",
307 "content.done",
308 "chunk",
309 ]
310 )
311 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
312 """\
313[
314 ParsedChoice[Location](
315 finish_reason='stop',
316 index=0,
317 logprobs=None,
318 message=ParsedChatCompletionMessage[Location](
319 audio=None,
320 content='{"city":"San Francisco","temperature":65,"units":"f"}',
321 function_call=None,
322 parsed=Location(city='San Francisco', temperature=65.0, units='f'),
323 refusal=None,
324 role='assistant',
325 tool_calls=[]
326 )
327 ),
328 ParsedChoice[Location](
329 finish_reason='stop',
330 index=1,
331 logprobs=None,
332 message=ParsedChatCompletionMessage[Location](
333 audio=None,
334 content='{"city":"San Francisco","temperature":61,"units":"f"}',
335 function_call=None,
336 parsed=Location(city='San Francisco', temperature=61.0, units='f'),
337 refusal=None,
338 role='assistant',
339 tool_calls=[]
340 )
341 ),
342 ParsedChoice[Location](
343 finish_reason='stop',
344 index=2,
345 logprobs=None,
346 message=ParsedChatCompletionMessage[Location](
347 audio=None,
348 content='{"city":"San Francisco","temperature":59,"units":"f"}',
349 function_call=None,
350 parsed=Location(city='San Francisco', temperature=59.0, units='f'),
351 refusal=None,
352 role='assistant',
353 tool_calls=[]
354 )
355 )
356]
357"""
358 )
359
360
361@pytest.mark.respx(base_url=base_url)
362def test_parse_max_tokens_reached(client: OpenAI, respx_mock: MockRouter) -> None:
363 class Location(BaseModel):
364 city: str
365 temperature: float
366 units: Literal["c", "f"]
367
368 with pytest.raises(openai.LengthFinishReasonError):
369 _make_stream_snapshot_request(
370 lambda c: c.beta.chat.completions.stream(
371 model="gpt-4o-2024-08-06",
372 messages=[
373 {
374 "role": "user",
375 "content": "What's the weather like in SF?",
376 },
377 ],
378 max_tokens=1,
379 response_format=Location,
380 ),
381 content_snapshot=snapshot(external("4cc50a6135d2*.bin")),
382 mock_client=client,
383 respx_mock=respx_mock,
384 )
385
386
387@pytest.mark.respx(base_url=base_url)
388def test_parse_pydantic_model_refusal(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
389 class Location(BaseModel):
390 city: str
391 temperature: float
392 units: Literal["c", "f"]
393
394 listener = _make_stream_snapshot_request(
395 lambda c: c.beta.chat.completions.stream(
396 model="gpt-4o-2024-08-06",
397 messages=[
398 {
399 "role": "user",
400 "content": "How do I make anthrax?",
401 },
402 ],
403 response_format=Location,
404 ),
405 content_snapshot=snapshot(external("173417d55340*.bin")),
406 mock_client=client,
407 respx_mock=respx_mock,
408 )
409
410 assert print_obj(listener.get_event_by_type("refusal.done"), monkeypatch) == snapshot("""\
411RefusalDoneEvent(refusal="I'm sorry, I can't assist with that request.", type='refusal.done')
412""")
413
414 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
415 """\
416[
417 ParsedChoice[Location](
418 finish_reason='stop',
419 index=0,
420 logprobs=None,
421 message=ParsedChatCompletionMessage[Location](
422 audio=None,
423 content=None,
424 function_call=None,
425 parsed=None,
426 refusal="I'm sorry, I can't assist with that request.",
427 role='assistant',
428 tool_calls=[]
429 )
430 )
431]
432"""
433 )
434
435
436@pytest.mark.respx(base_url=base_url)
437def test_content_logprobs_events(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
438 listener = _make_stream_snapshot_request(
439 lambda c: c.beta.chat.completions.stream(
440 model="gpt-4o-2024-08-06",
441 messages=[
442 {
443 "role": "user",
444 "content": "Say foo",
445 },
446 ],
447 logprobs=True,
448 ),
449 content_snapshot=snapshot(external("83b060bae42e*.bin")),
450 mock_client=client,
451 respx_mock=respx_mock,
452 )
453
454 assert print_obj([e for e in listener.events if e.type.startswith("logprobs")], monkeypatch) == snapshot("""\
455[
456 LogprobsContentDeltaEvent(
457 content=[
458 ChatCompletionTokenLogprob(bytes=[70, 111, 111], logprob=-0.0025094282, token='Foo', top_logprobs=[])
459 ],
460 snapshot=[
461 ChatCompletionTokenLogprob(bytes=[70, 111, 111], logprob=-0.0025094282, token='Foo', top_logprobs=[])
462 ],
463 type='logprobs.content.delta'
464 ),
465 LogprobsContentDeltaEvent(
466 content=[ChatCompletionTokenLogprob(bytes=[33], logprob=-0.26638845, token='!', top_logprobs=[])],
467 snapshot=[
468 ChatCompletionTokenLogprob(bytes=[70, 111, 111], logprob=-0.0025094282, token='Foo', top_logprobs=[]),
469 ChatCompletionTokenLogprob(bytes=[33], logprob=-0.26638845, token='!', top_logprobs=[])
470 ],
471 type='logprobs.content.delta'
472 ),
473 LogprobsContentDoneEvent(
474 content=[
475 ChatCompletionTokenLogprob(bytes=[70, 111, 111], logprob=-0.0025094282, token='Foo', top_logprobs=[]),
476 ChatCompletionTokenLogprob(bytes=[33], logprob=-0.26638845, token='!', top_logprobs=[])
477 ],
478 type='logprobs.content.done'
479 )
480]
481""")
482
483 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot("""\
484[
485 ParsedChoice[NoneType](
486 finish_reason='stop',
487 index=0,
488 logprobs=ChoiceLogprobs(
489 content=[
490 ChatCompletionTokenLogprob(bytes=[70, 111, 111], logprob=-0.0025094282, token='Foo', top_logprobs=[]),
491 ChatCompletionTokenLogprob(bytes=[33], logprob=-0.26638845, token='!', top_logprobs=[])
492 ],
493 refusal=None
494 ),
495 message=ParsedChatCompletionMessage[NoneType](
496 audio=None,
497 content='Foo!',
498 function_call=None,
499 parsed=None,
500 refusal=None,
501 role='assistant',
502 tool_calls=[]
503 )
504 )
505]
506""")
507
508
509@pytest.mark.respx(base_url=base_url)
510def test_refusal_logprobs_events(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
511 class Location(BaseModel):
512 city: str
513 temperature: float
514 units: Literal["c", "f"]
515
516 listener = _make_stream_snapshot_request(
517 lambda c: c.beta.chat.completions.stream(
518 model="gpt-4o-2024-08-06",
519 messages=[
520 {
521 "role": "user",
522 "content": "How do I make anthrax?",
523 },
524 ],
525 logprobs=True,
526 response_format=Location,
527 ),
528 content_snapshot=snapshot(external("569c877e6942*.bin")),
529 mock_client=client,
530 respx_mock=respx_mock,
531 )
532
533 assert print_obj([e.type for e in listener.events if e.type.startswith("logprobs")], monkeypatch) == snapshot("""\
534[
535 'logprobs.refusal.delta',
536 'logprobs.refusal.delta',
537 'logprobs.refusal.delta',
538 'logprobs.refusal.delta',
539 'logprobs.refusal.delta',
540 'logprobs.refusal.delta',
541 'logprobs.refusal.delta',
542 'logprobs.refusal.delta',
543 'logprobs.refusal.delta',
544 'logprobs.refusal.delta',
545 'logprobs.refusal.delta',
546 'logprobs.refusal.done'
547]
548""")
549
550 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot("""\
551[
552 ParsedChoice[Location](
553 finish_reason='stop',
554 index=0,
555 logprobs=ChoiceLogprobs(
556 content=None,
557 refusal=[
558 ChatCompletionTokenLogprob(bytes=[73, 39, 109], logprob=-0.0012038043, token="I'm", top_logprobs=[]),
559 ChatCompletionTokenLogprob(
560 bytes=[32, 118, 101, 114, 121],
561 logprob=-0.8438816,
562 token=' very',
563 top_logprobs=[]
564 ),
565 ChatCompletionTokenLogprob(
566 bytes=[32, 115, 111, 114, 114, 121],
567 logprob=-3.4121115e-06,
568 token=' sorry',
569 top_logprobs=[]
570 ),
571 ChatCompletionTokenLogprob(bytes=[44], logprob=-3.3809047e-05, token=',', top_logprobs=[]),
572 ChatCompletionTokenLogprob(
573 bytes=[32, 98, 117, 116],
574 logprob=-0.038048144,
575 token=' but',
576 top_logprobs=[]
577 ),
578 ChatCompletionTokenLogprob(bytes=[32, 73], logprob=-0.0016109125, token=' I', top_logprobs=[]),
579 ChatCompletionTokenLogprob(
580 bytes=[32, 99, 97, 110, 39, 116],
581 logprob=-0.0073532974,
582 token=" can't",
583 top_logprobs=[]
584 ),
585 ChatCompletionTokenLogprob(
586 bytes=[32, 97, 115, 115, 105, 115, 116],
587 logprob=-0.0020837625,
588 token=' assist',
589 top_logprobs=[]
590 ),
591 ChatCompletionTokenLogprob(
592 bytes=[32, 119, 105, 116, 104],
593 logprob=-0.00318354,
594 token=' with',
595 top_logprobs=[]
596 ),
597 ChatCompletionTokenLogprob(
598 bytes=[32, 116, 104, 97, 116],
599 logprob=-0.0017186158,
600 token=' that',
601 top_logprobs=[]
602 ),
603 ChatCompletionTokenLogprob(bytes=[46], logprob=-0.57687104, token='.', top_logprobs=[])
604 ]
605 ),
606 message=ParsedChatCompletionMessage[Location](
607 audio=None,
608 content=None,
609 function_call=None,
610 parsed=None,
611 refusal="I'm very sorry, but I can't assist with that.",
612 role='assistant',
613 tool_calls=[]
614 )
615 )
616]
617""")
618
619
620@pytest.mark.respx(base_url=base_url)
621def test_parse_pydantic_tool(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
622 class GetWeatherArgs(BaseModel):
623 city: str
624 country: str
625 units: Literal["c", "f"] = "c"
626
627 listener = _make_stream_snapshot_request(
628 lambda c: c.beta.chat.completions.stream(
629 model="gpt-4o-2024-08-06",
630 messages=[
631 {
632 "role": "user",
633 "content": "What's the weather like in Edinburgh?",
634 },
635 ],
636 tools=[
637 openai.pydantic_function_tool(GetWeatherArgs),
638 ],
639 ),
640 content_snapshot=snapshot(external("c6aa7e397b71*.bin")),
641 mock_client=client,
642 respx_mock=respx_mock,
643 )
644
645 assert print_obj(listener.stream.current_completion_snapshot.choices, monkeypatch) == snapshot(
646 """\
647[
648 ParsedChoice[object](
649 finish_reason='tool_calls',
650 index=0,
651 logprobs=None,
652 message=ParsedChatCompletionMessage[object](
653 audio=None,
654 content=None,
655 function_call=None,
656 parsed=None,
657 refusal=None,
658 role='assistant',
659 tool_calls=[
660 ParsedFunctionToolCall(
661 function=ParsedFunction(
662 arguments='{"city":"Edinburgh","country":"UK","units":"c"}',
663 name='GetWeatherArgs',
664 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='UK', units='c')
665 ),
666 id='call_c91SqDXlYFuETYv8mUHzz6pp',
667 index=0,
668 type='function'
669 )
670 ]
671 )
672 )
673]
674"""
675 )
676
677 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
678 """\
679[
680 ParsedChoice[NoneType](
681 finish_reason='tool_calls',
682 index=0,
683 logprobs=None,
684 message=ParsedChatCompletionMessage[NoneType](
685 audio=None,
686 content=None,
687 function_call=None,
688 parsed=None,
689 refusal=None,
690 role='assistant',
691 tool_calls=[
692 ParsedFunctionToolCall(
693 function=ParsedFunction(
694 arguments='{"city":"Edinburgh","country":"UK","units":"c"}',
695 name='GetWeatherArgs',
696 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='UK', units='c')
697 ),
698 id='call_c91SqDXlYFuETYv8mUHzz6pp',
699 index=0,
700 type='function'
701 )
702 ]
703 )
704 )
705]
706"""
707 )
708
709
710@pytest.mark.respx(base_url=base_url)
711def test_parse_multiple_pydantic_tools(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
712 class GetWeatherArgs(BaseModel):
713 """Get the temperature for the given country/city combo"""
714
715 city: str
716 country: str
717 units: Literal["c", "f"] = "c"
718
719 class GetStockPrice(BaseModel):
720 ticker: str
721 exchange: str
722
723 listener = _make_stream_snapshot_request(
724 lambda c: c.beta.chat.completions.stream(
725 model="gpt-4o-2024-08-06",
726 messages=[
727 {
728 "role": "user",
729 "content": "What's the weather like in Edinburgh?",
730 },
731 {
732 "role": "user",
733 "content": "What's the price of AAPL?",
734 },
735 ],
736 tools=[
737 openai.pydantic_function_tool(GetWeatherArgs),
738 openai.pydantic_function_tool(
739 GetStockPrice, name="get_stock_price", description="Fetch the latest price for a given ticker"
740 ),
741 ],
742 ),
743 content_snapshot=snapshot(external("f82268f2fefd*.bin")),
744 mock_client=client,
745 respx_mock=respx_mock,
746 )
747
748 assert print_obj(listener.stream.current_completion_snapshot.choices, monkeypatch) == snapshot(
749 """\
750[
751 ParsedChoice[object](
752 finish_reason='tool_calls',
753 index=0,
754 logprobs=None,
755 message=ParsedChatCompletionMessage[object](
756 audio=None,
757 content=None,
758 function_call=None,
759 parsed=None,
760 refusal=None,
761 role='assistant',
762 tool_calls=[
763 ParsedFunctionToolCall(
764 function=ParsedFunction(
765 arguments='{"city": "Edinburgh", "country": "GB", "units": "c"}',
766 name='GetWeatherArgs',
767 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='GB', units='c')
768 ),
769 id='call_JMW1whyEaYG438VE1OIflxA2',
770 index=0,
771 type='function'
772 ),
773 ParsedFunctionToolCall(
774 function=ParsedFunction(
775 arguments='{"ticker": "AAPL", "exchange": "NASDAQ"}',
776 name='get_stock_price',
777 parsed_arguments=GetStockPrice(exchange='NASDAQ', ticker='AAPL')
778 ),
779 id='call_DNYTawLBoN8fj3KN6qU9N1Ou',
780 index=1,
781 type='function'
782 )
783 ]
784 )
785 )
786]
787"""
788 )
789 completion = listener.stream.get_final_completion()
790 assert print_obj(completion.choices[0].message.tool_calls, monkeypatch) == snapshot(
791 """\
792[
793 ParsedFunctionToolCall(
794 function=ParsedFunction(
795 arguments='{"city": "Edinburgh", "country": "GB", "units": "c"}',
796 name='GetWeatherArgs',
797 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='GB', units='c')
798 ),
799 id='call_JMW1whyEaYG438VE1OIflxA2',
800 index=0,
801 type='function'
802 ),
803 ParsedFunctionToolCall(
804 function=ParsedFunction(
805 arguments='{"ticker": "AAPL", "exchange": "NASDAQ"}',
806 name='get_stock_price',
807 parsed_arguments=GetStockPrice(exchange='NASDAQ', ticker='AAPL')
808 ),
809 id='call_DNYTawLBoN8fj3KN6qU9N1Ou',
810 index=1,
811 type='function'
812 )
813]
814"""
815 )
816
817
818@pytest.mark.respx(base_url=base_url)
819def test_parse_strict_tools(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
820 listener = _make_stream_snapshot_request(
821 lambda c: c.beta.chat.completions.stream(
822 model="gpt-4o-2024-08-06",
823 messages=[
824 {
825 "role": "user",
826 "content": "What's the weather like in SF?",
827 },
828 ],
829 tools=[
830 {
831 "type": "function",
832 "function": {
833 "name": "get_weather",
834 "parameters": {
835 "type": "object",
836 "properties": {
837 "city": {"type": "string"},
838 "state": {"type": "string"},
839 },
840 "required": [
841 "city",
842 "state",
843 ],
844 "additionalProperties": False,
845 },
846 "strict": True,
847 },
848 }
849 ],
850 ),
851 content_snapshot=snapshot(external("a247c49c5fcd*.bin")),
852 mock_client=client,
853 respx_mock=respx_mock,
854 )
855
856 assert print_obj(listener.stream.current_completion_snapshot.choices, monkeypatch) == snapshot(
857 """\
858[
859 ParsedChoice[object](
860 finish_reason='tool_calls',
861 index=0,
862 logprobs=None,
863 message=ParsedChatCompletionMessage[object](
864 audio=None,
865 content=None,
866 function_call=None,
867 parsed=None,
868 refusal=None,
869 role='assistant',
870 tool_calls=[
871 ParsedFunctionToolCall(
872 function=ParsedFunction(
873 arguments='{"city":"San Francisco","state":"CA"}',
874 name='get_weather',
875 parsed_arguments={'city': 'San Francisco', 'state': 'CA'}
876 ),
877 id='call_CTf1nWJLqSeRgDqaCG27xZ74',
878 index=0,
879 type='function'
880 )
881 ]
882 )
883 )
884]
885"""
886 )
887
888
889@pytest.mark.respx(base_url=base_url)
890def test_non_pydantic_response_format(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
891 listener = _make_stream_snapshot_request(
892 lambda c: c.beta.chat.completions.stream(
893 model="gpt-4o-2024-08-06",
894 messages=[
895 {
896 "role": "user",
897 "content": "What's the weather like in SF? Give me any JSON back",
898 },
899 ],
900 response_format={"type": "json_object"},
901 ),
902 content_snapshot=snapshot(external("d61558011839*.bin")),
903 mock_client=client,
904 respx_mock=respx_mock,
905 )
906
907 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
908 """\
909[
910 ParsedChoice[NoneType](
911 finish_reason='stop',
912 index=0,
913 logprobs=None,
914 message=ParsedChatCompletionMessage[NoneType](
915 audio=None,
916 content='\\n {\\n "location": "San Francisco, CA",\\n "weather": {\\n "temperature": "18°C",\\n
917"condition": "Partly Cloudy",\\n "humidity": "72%",\\n "windSpeed": "15 km/h",\\n "windDirection": "NW"\\n
918},\\n "forecast": [\\n {\\n "day": "Monday",\\n "high": "20°C",\\n "low": "14°C",\\n
919"condition": "Sunny"\\n },\\n {\\n "day": "Tuesday",\\n "high": "19°C",\\n "low": "15°C",\\n
920"condition": "Mostly Cloudy"\\n },\\n {\\n "day": "Wednesday",\\n "high": "18°C",\\n "low":
921"14°C",\\n "condition": "Cloudy"\\n }\\n ]\\n }\\n',
922 function_call=None,
923 parsed=None,
924 refusal=None,
925 role='assistant',
926 tool_calls=[]
927 )
928 )
929]
930"""
931 )
932
933
934@pytest.mark.respx(base_url=base_url)
935def test_allows_non_strict_tools_but_no_parsing(
936 client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch
937) -> None:
938 listener = _make_stream_snapshot_request(
939 lambda c: c.beta.chat.completions.stream(
940 model="gpt-4o-2024-08-06",
941 messages=[{"role": "user", "content": "what's the weather in NYC?"}],
942 tools=[
943 {
944 "type": "function",
945 "function": {
946 "name": "get_weather",
947 "parameters": {"type": "object", "properties": {"city": {"type": "string"}}},
948 },
949 }
950 ],
951 ),
952 content_snapshot=snapshot(external("2018feb66ae1*.bin")),
953 mock_client=client,
954 respx_mock=respx_mock,
955 )
956
957 assert print_obj(listener.get_event_by_type("tool_calls.function.arguments.done"), monkeypatch) == snapshot("""\
958FunctionToolCallArgumentsDoneEvent(
959 arguments='{"city":"New York City"}',
960 index=0,
961 name='get_weather',
962 parsed_arguments=None,
963 type='tool_calls.function.arguments.done'
964)
965""")
966
967 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
968 """\
969[
970 ParsedChoice[NoneType](
971 finish_reason='tool_calls',
972 index=0,
973 logprobs=None,
974 message=ParsedChatCompletionMessage[NoneType](
975 audio=None,
976 content=None,
977 function_call=None,
978 parsed=None,
979 refusal=None,
980 role='assistant',
981 tool_calls=[
982 ParsedFunctionToolCall(
983 function=ParsedFunction(
984 arguments='{"city":"New York City"}',
985 name='get_weather',
986 parsed_arguments=None
987 ),
988 id='call_4XzlGBLtUe9dy3GVNV4jhq7h',
989 index=0,
990 type='function'
991 )
992 ]
993 )
994 )
995]
996"""
997 )
998
999
1000@pytest.mark.parametrize("sync", [True, False], ids=["sync", "async"])
1001def test_stream_method_in_sync(sync: bool, client: OpenAI, async_client: AsyncOpenAI) -> None:
1002 checking_client: OpenAI | AsyncOpenAI = client if sync else async_client
1003
1004 assert_signatures_in_sync(
1005 checking_client.chat.completions.create,
1006 checking_client.beta.chat.completions.stream,
1007 exclude_params={"response_format", "stream"},
1008 )
1009
1010
1011class StreamListener(Generic[ResponseFormatT]):
1012 def __init__(self, stream: ChatCompletionStream[ResponseFormatT]) -> None:
1013 self.stream = stream
1014 self.events: list[ChatCompletionStreamEvent[ResponseFormatT]] = []
1015
1016 def __iter__(self) -> Iterator[ChatCompletionStreamEvent[ResponseFormatT]]:
1017 for event in self.stream:
1018 self.events.append(event)
1019 yield event
1020
1021 @overload
1022 def get_event_by_type(self, event_type: Literal["content.done"]) -> ContentDoneEvent[ResponseFormatT] | None: ...
1023
1024 @overload
1025 def get_event_by_type(self, event_type: str) -> ChatCompletionStreamEvent[ResponseFormatT] | None: ...
1026
1027 def get_event_by_type(self, event_type: str) -> ChatCompletionStreamEvent[ResponseFormatT] | None:
1028 return next((e for e in self.events if e.type == event_type), None)
1029
1030
1031def _make_stream_snapshot_request(
1032 func: Callable[[OpenAI], ChatCompletionStreamManager[ResponseFormatT]],
1033 *,
1034 content_snapshot: Any,
1035 respx_mock: MockRouter,
1036 mock_client: OpenAI,
1037 on_event: Callable[[ChatCompletionStream[ResponseFormatT], ChatCompletionStreamEvent[ResponseFormatT]], Any]
1038 | None = None,
1039) -> StreamListener[ResponseFormatT]:
1040 live = os.environ.get("OPENAI_LIVE") == "1"
1041 if live:
1042
1043 def _on_response(response: httpx.Response) -> None:
1044 # update the content snapshot
1045 assert outsource(response.read()) == content_snapshot
1046
1047 respx_mock.stop()
1048
1049 client = OpenAI(
1050 http_client=httpx.Client(
1051 event_hooks={
1052 "response": [_on_response],
1053 }
1054 )
1055 )
1056 else:
1057 respx_mock.post("/chat/completions").mock(
1058 return_value=httpx.Response(
1059 200,
1060 content=content_snapshot._old_value._load_value(),
1061 headers={"content-type": "text/event-stream"},
1062 )
1063 )
1064
1065 client = mock_client
1066
1067 with func(client) as stream:
1068 listener = StreamListener(stream)
1069
1070 for event in listener:
1071 if on_event:
1072 on_event(stream, event)
1073
1074 if live:
1075 client.close()
1076
1077 return listener
1078