GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

openai/openai-python

Public

mirrored from https://github.com/openai/openai-pythonAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
bf1ca86cf392eb0ffed1e146937c5d73d8a568f0

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openai-python/tests/lib/chat

tests/lib/chat/test_completions_streaming.py

1047lines · modecode

RawDownload
Latest commit unavailable.
1from __future__ import annotations
2
3import os
4from typing import Any, Generic, Callable, Iterator, cast, overload
5from typing_extensions import Literal, TypeVar
6
7import rich
8import httpx
9import pytest
10from respx import MockRouter
11from pydantic import BaseModel
12from inline_snapshot import external, snapshot, outsource
13
14import openai
15from openai import OpenAI, AsyncOpenAI
16from openai._utils import assert_signatures_in_sync
17from openai._compat import model_copy
18from openai.lib.streaming.chat import (
19 ContentDoneEvent,
20 ChatCompletionStream,
21 ChatCompletionStreamEvent,
22 ChatCompletionStreamManager,
23 ParsedChatCompletionSnapshot,
24)
25from openai.lib._parsing._completions import ResponseFormatT
26
27from ._utils import print_obj
28from ...conftest import base_url
29
30_T = TypeVar("_T")
31
32# all the snapshots in this file are auto-generated from the live API
33#
34# you can update them with
35#
36# `OPENAI_LIVE=1 pytest --inline-snapshot=fix`
37
38
39@pytest.mark.respx(base_url=base_url)
40def test_parse_nothing(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
41 listener = _make_stream_snapshot_request(
42 lambda c: c.beta.chat.completions.stream(
43 model="gpt-4o-2024-08-06",
44 messages=[
45 {
46 "role": "user",
47 "content": "What's the weather like in SF?",
48 },
49 ],
50 ),
51 content_snapshot=snapshot(external("b9d6bee9f9b8*.bin")),
52 mock_client=client,
53 respx_mock=respx_mock,
54 )
55
56 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
57 """\
58[
59 ParsedChoice[NoneType](
60 finish_reason='stop',
61 index=0,
62 logprobs=None,
63 message=ParsedChatCompletionMessage[NoneType](
64 content="I'm unable to provide real-time weather updates. To get the latest weather information for San
65Francisco, I recommend checking a reliable weather website or using a weather app.",
66 function_call=None,
67 parsed=None,
68 refusal=None,
69 role='assistant',
70 tool_calls=[]
71 )
72 )
73]
74"""
75 )
76 assert print_obj(listener.get_event_by_type("content.done"), monkeypatch) == snapshot(
77 """\
78ContentDoneEvent[NoneType](
79 content="I'm unable to provide real-time weather updates. To get the latest weather information for San Francisco, I
80recommend checking a reliable weather website or using a weather app.",
81 parsed=None,
82 type='content.done'
83)
84"""
85 )
86
87
88@pytest.mark.respx(base_url=base_url)
89def test_parse_pydantic_model(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
90 class Location(BaseModel):
91 city: str
92 temperature: float
93 units: Literal["c", "f"]
94
95 done_snapshots: list[ParsedChatCompletionSnapshot] = []
96
97 def on_event(stream: ChatCompletionStream[Location], event: ChatCompletionStreamEvent[Location]) -> None:
98 if event.type == "content.done":
99 done_snapshots.append(model_copy(stream.current_completion_snapshot, deep=True))
100
101 listener = _make_stream_snapshot_request(
102 lambda c: c.beta.chat.completions.stream(
103 model="gpt-4o-2024-08-06",
104 messages=[
105 {
106 "role": "user",
107 "content": "What's the weather like in SF?",
108 },
109 ],
110 response_format=Location,
111 ),
112 content_snapshot=snapshot(external("ea9a417d533b*.bin")),
113 mock_client=client,
114 respx_mock=respx_mock,
115 on_event=on_event,
116 )
117
118 assert len(done_snapshots) == 1
119 assert isinstance(done_snapshots[0].choices[0].message.parsed, Location)
120
121 for event in reversed(listener.events):
122 if event.type == "content.delta":
123 data = cast(Any, event.parsed)
124 assert isinstance(data["city"], str), data
125 assert isinstance(data["temperature"], (int, float)), data
126 assert isinstance(data["units"], str), data
127 break
128 else:
129 rich.print(listener.events)
130 raise AssertionError("Did not find a `content.delta` event")
131
132 assert print_obj(listener.stream.get_final_completion(), monkeypatch) == snapshot(
133 """\
134ParsedChatCompletion[Location](
135 choices=[
136 ParsedChoice[Location](
137 finish_reason='stop',
138 index=0,
139 logprobs=None,
140 message=ParsedChatCompletionMessage[Location](
141 content='{"city":"San Francisco","temperature":63,"units":"f"}',
142 function_call=None,
143 parsed=Location(city='San Francisco', temperature=63.0, units='f'),
144 refusal=None,
145 role='assistant',
146 tool_calls=[]
147 )
148 )
149 ],
150 created=1722934250,
151 id='chatcmpl-9tAC2Fr44W8e4GakwKuKSSsFPhISv',
152 model='gpt-4o-so',
153 object='chat.completion',
154 service_tier=None,
155 system_fingerprint='fp_e1a05a1dce',
156 usage=CompletionUsage(completion_tokens=14, prompt_tokens=17, total_tokens=31)
157)
158"""
159 )
160 assert print_obj(listener.get_event_by_type("content.done"), monkeypatch) == snapshot(
161 """\
162ContentDoneEvent[Location](
163 content='{"city":"San Francisco","temperature":63,"units":"f"}',
164 parsed=Location(city='San Francisco', temperature=63.0, units='f'),
165 type='content.done'
166)
167"""
168 )
169
170
171@pytest.mark.respx(base_url=base_url)
172def test_parse_pydantic_model_multiple_choices(
173 client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch
174) -> None:
175 class Location(BaseModel):
176 city: str
177 temperature: float
178 units: Literal["c", "f"]
179
180 listener = _make_stream_snapshot_request(
181 lambda c: c.beta.chat.completions.stream(
182 model="gpt-4o-2024-08-06",
183 messages=[
184 {
185 "role": "user",
186 "content": "What's the weather like in SF?",
187 },
188 ],
189 n=3,
190 response_format=Location,
191 ),
192 content_snapshot=snapshot(external("1437bd06a9d5*.bin")),
193 mock_client=client,
194 respx_mock=respx_mock,
195 )
196
197 assert [e.type for e in listener.events] == snapshot(
198 [
199 "chunk",
200 "content.delta",
201 "chunk",
202 "content.delta",
203 "chunk",
204 "content.delta",
205 "chunk",
206 "content.delta",
207 "chunk",
208 "content.delta",
209 "chunk",
210 "content.delta",
211 "chunk",
212 "content.delta",
213 "chunk",
214 "content.delta",
215 "chunk",
216 "content.delta",
217 "chunk",
218 "content.delta",
219 "chunk",
220 "content.delta",
221 "chunk",
222 "content.delta",
223 "chunk",
224 "content.delta",
225 "chunk",
226 "content.delta",
227 "chunk",
228 "content.delta",
229 "chunk",
230 "content.delta",
231 "chunk",
232 "content.delta",
233 "chunk",
234 "content.delta",
235 "chunk",
236 "content.delta",
237 "chunk",
238 "content.delta",
239 "chunk",
240 "content.delta",
241 "chunk",
242 "content.delta",
243 "chunk",
244 "content.delta",
245 "chunk",
246 "content.delta",
247 "chunk",
248 "content.delta",
249 "chunk",
250 "content.delta",
251 "chunk",
252 "content.delta",
253 "chunk",
254 "content.delta",
255 "chunk",
256 "content.delta",
257 "chunk",
258 "content.delta",
259 "chunk",
260 "content.delta",
261 "chunk",
262 "content.delta",
263 "chunk",
264 "content.delta",
265 "chunk",
266 "content.delta",
267 "chunk",
268 "content.delta",
269 "chunk",
270 "content.delta",
271 "chunk",
272 "content.delta",
273 "chunk",
274 "content.delta",
275 "chunk",
276 "content.delta",
277 "chunk",
278 "content.delta",
279 "chunk",
280 "content.delta",
281 "chunk",
282 "content.delta",
283 "chunk",
284 "content.delta",
285 "chunk",
286 "content.delta",
287 "chunk",
288 "content.delta",
289 "chunk",
290 "content.done",
291 "chunk",
292 "content.done",
293 "chunk",
294 "content.done",
295 "chunk",
296 ]
297 )
298 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
299 """\
300[
301 ParsedChoice[Location](
302 finish_reason='stop',
303 index=0,
304 logprobs=None,
305 message=ParsedChatCompletionMessage[Location](
306 content='{"city":"San Francisco","temperature":64,"units":"f"}',
307 function_call=None,
308 parsed=Location(city='San Francisco', temperature=64.0, units='f'),
309 refusal=None,
310 role='assistant',
311 tool_calls=[]
312 )
313 ),
314 ParsedChoice[Location](
315 finish_reason='stop',
316 index=1,
317 logprobs=None,
318 message=ParsedChatCompletionMessage[Location](
319 content='{"city":"San Francisco","temperature":68,"units":"f"}',
320 function_call=None,
321 parsed=Location(city='San Francisco', temperature=68.0, units='f'),
322 refusal=None,
323 role='assistant',
324 tool_calls=[]
325 )
326 ),
327 ParsedChoice[Location](
328 finish_reason='stop',
329 index=2,
330 logprobs=None,
331 message=ParsedChatCompletionMessage[Location](
332 content='{"city":"San Francisco","temperature":64,"units":"f"}',
333 function_call=None,
334 parsed=Location(city='San Francisco', temperature=64.0, units='f'),
335 refusal=None,
336 role='assistant',
337 tool_calls=[]
338 )
339 )
340]
341"""
342 )
343
344
345@pytest.mark.respx(base_url=base_url)
346def test_parse_max_tokens_reached(client: OpenAI, respx_mock: MockRouter) -> None:
347 class Location(BaseModel):
348 city: str
349 temperature: float
350 units: Literal["c", "f"]
351
352 with pytest.raises(openai.LengthFinishReasonError):
353 _make_stream_snapshot_request(
354 lambda c: c.beta.chat.completions.stream(
355 model="gpt-4o-2024-08-06",
356 messages=[
357 {
358 "role": "user",
359 "content": "What's the weather like in SF?",
360 },
361 ],
362 max_tokens=1,
363 response_format=Location,
364 ),
365 content_snapshot=snapshot(external("7ae6c1a2631b*.bin")),
366 mock_client=client,
367 respx_mock=respx_mock,
368 )
369
370
371@pytest.mark.respx(base_url=base_url)
372def test_parse_pydantic_model_refusal(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
373 class Location(BaseModel):
374 city: str
375 temperature: float
376 units: Literal["c", "f"]
377
378 listener = _make_stream_snapshot_request(
379 lambda c: c.beta.chat.completions.stream(
380 model="gpt-4o-2024-08-06",
381 messages=[
382 {
383 "role": "user",
384 "content": "How do I make anthrax?",
385 },
386 ],
387 response_format=Location,
388 ),
389 content_snapshot=snapshot(external("d79326933c15*.bin")),
390 mock_client=client,
391 respx_mock=respx_mock,
392 )
393
394 assert print_obj(listener.get_event_by_type("refusal.done"), monkeypatch) == snapshot("""\
395RefusalDoneEvent(refusal="I'm very sorry, but I can't assist with that request.", type='refusal.done')
396""")
397
398 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
399 """\
400[
401 ParsedChoice[Location](
402 finish_reason='stop',
403 index=0,
404 logprobs=None,
405 message=ParsedChatCompletionMessage[Location](
406 content=None,
407 function_call=None,
408 parsed=None,
409 refusal="I'm very sorry, but I can't assist with that request.",
410 role='assistant',
411 tool_calls=[]
412 )
413 )
414]
415"""
416 )
417
418
419@pytest.mark.respx(base_url=base_url)
420def test_content_logprobs_events(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
421 listener = _make_stream_snapshot_request(
422 lambda c: c.beta.chat.completions.stream(
423 model="gpt-4o-2024-08-06",
424 messages=[
425 {
426 "role": "user",
427 "content": "Say foo",
428 },
429 ],
430 logprobs=True,
431 ),
432 content_snapshot=snapshot(external("70c7df71ce72*.bin")),
433 mock_client=client,
434 respx_mock=respx_mock,
435 )
436
437 assert print_obj([e for e in listener.events if e.type.startswith("logprobs")], monkeypatch) == snapshot("""\
438[
439 LogprobsContentDeltaEvent(
440 content=[ChatCompletionTokenLogprob(bytes=[70, 111, 111], logprob=-0.006764991, token='Foo', top_logprobs=[])],
441 snapshot=[
442 ChatCompletionTokenLogprob(bytes=[70, 111, 111], logprob=-0.006764991, token='Foo', top_logprobs=[])
443 ],
444 type='logprobs.content.delta'
445 ),
446 LogprobsContentDeltaEvent(
447 content=[ChatCompletionTokenLogprob(bytes=[33], logprob=-0.31380808, token='!', top_logprobs=[])],
448 snapshot=[
449 ChatCompletionTokenLogprob(bytes=[70, 111, 111], logprob=-0.006764991, token='Foo', top_logprobs=[]),
450 ChatCompletionTokenLogprob(bytes=[33], logprob=-0.31380808, token='!', top_logprobs=[])
451 ],
452 type='logprobs.content.delta'
453 ),
454 LogprobsContentDoneEvent(
455 content=[
456 ChatCompletionTokenLogprob(bytes=[70, 111, 111], logprob=-0.006764991, token='Foo', top_logprobs=[]),
457 ChatCompletionTokenLogprob(bytes=[33], logprob=-0.31380808, token='!', top_logprobs=[])
458 ],
459 type='logprobs.content.done'
460 )
461]
462""")
463
464 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot("""\
465[
466 ParsedChoice[NoneType](
467 finish_reason='stop',
468 index=0,
469 logprobs=ChoiceLogprobs(
470 content=[
471 ChatCompletionTokenLogprob(bytes=[70, 111, 111], logprob=-0.006764991, token='Foo', top_logprobs=[]),
472 ChatCompletionTokenLogprob(bytes=[33], logprob=-0.31380808, token='!', top_logprobs=[])
473 ],
474 refusal=None
475 ),
476 message=ParsedChatCompletionMessage[NoneType](
477 content='Foo!',
478 function_call=None,
479 parsed=None,
480 refusal=None,
481 role='assistant',
482 tool_calls=[]
483 )
484 )
485]
486""")
487
488
489@pytest.mark.respx(base_url=base_url)
490def test_refusal_logprobs_events(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
491 class Location(BaseModel):
492 city: str
493 temperature: float
494 units: Literal["c", "f"]
495
496 listener = _make_stream_snapshot_request(
497 lambda c: c.beta.chat.completions.stream(
498 model="gpt-4o-2024-08-06",
499 messages=[
500 {
501 "role": "user",
502 "content": "How do I make anthrax?",
503 },
504 ],
505 logprobs=True,
506 response_format=Location,
507 ),
508 content_snapshot=snapshot(external("cb77dc69b6c8*.bin")),
509 mock_client=client,
510 respx_mock=respx_mock,
511 )
512
513 assert print_obj([e.type for e in listener.events if e.type.startswith("logprobs")], monkeypatch) == snapshot("""\
514[
515 'logprobs.refusal.delta',
516 'logprobs.refusal.delta',
517 'logprobs.refusal.delta',
518 'logprobs.refusal.delta',
519 'logprobs.refusal.delta',
520 'logprobs.refusal.delta',
521 'logprobs.refusal.delta',
522 'logprobs.refusal.delta',
523 'logprobs.refusal.delta',
524 'logprobs.refusal.delta',
525 'logprobs.refusal.delta',
526 'logprobs.refusal.done'
527]
528""")
529
530 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot("""\
531[
532 ParsedChoice[Location](
533 finish_reason='stop',
534 index=0,
535 logprobs=ChoiceLogprobs(
536 content=None,
537 refusal=[
538 ChatCompletionTokenLogprob(bytes=[73, 39, 109], logprob=-0.0010472201, token="I'm", top_logprobs=[]),
539 ChatCompletionTokenLogprob(
540 bytes=[32, 118, 101, 114, 121],
541 logprob=-0.7292482,
542 token=' very',
543 top_logprobs=[]
544 ),
545 ChatCompletionTokenLogprob(
546 bytes=[32, 115, 111, 114, 114, 121],
547 logprob=-5.080963e-06,
548 token=' sorry',
549 top_logprobs=[]
550 ),
551 ChatCompletionTokenLogprob(bytes=[44], logprob=-4.048445e-05, token=',', top_logprobs=[]),
552 ChatCompletionTokenLogprob(
553 bytes=[32, 98, 117, 116],
554 logprob=-0.038046427,
555 token=' but',
556 top_logprobs=[]
557 ),
558 ChatCompletionTokenLogprob(bytes=[32, 73], logprob=-0.0019351852, token=' I', top_logprobs=[]),
559 ChatCompletionTokenLogprob(
560 bytes=[32, 99, 97, 110, 39, 116],
561 logprob=-0.008995773,
562 token=" can't",
563 top_logprobs=[]
564 ),
565 ChatCompletionTokenLogprob(
566 bytes=[32, 97, 115, 115, 105, 115, 116],
567 logprob=-0.0033510819,
568 token=' assist',
569 top_logprobs=[]
570 ),
571 ChatCompletionTokenLogprob(
572 bytes=[32, 119, 105, 116, 104],
573 logprob=-0.0036033941,
574 token=' with',
575 top_logprobs=[]
576 ),
577 ChatCompletionTokenLogprob(
578 bytes=[32, 116, 104, 97, 116],
579 logprob=-0.0015974608,
580 token=' that',
581 top_logprobs=[]
582 ),
583 ChatCompletionTokenLogprob(bytes=[46], logprob=-0.6339823, token='.', top_logprobs=[])
584 ]
585 ),
586 message=ParsedChatCompletionMessage[Location](
587 content=None,
588 function_call=None,
589 parsed=None,
590 refusal="I'm very sorry, but I can't assist with that.",
591 role='assistant',
592 tool_calls=[]
593 )
594 )
595]
596""")
597
598
599@pytest.mark.respx(base_url=base_url)
600def test_parse_pydantic_tool(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
601 class GetWeatherArgs(BaseModel):
602 city: str
603 country: str
604 units: Literal["c", "f"] = "c"
605
606 listener = _make_stream_snapshot_request(
607 lambda c: c.beta.chat.completions.stream(
608 model="gpt-4o-2024-08-06",
609 messages=[
610 {
611 "role": "user",
612 "content": "What's the weather like in Edinburgh?",
613 },
614 ],
615 tools=[
616 openai.pydantic_function_tool(GetWeatherArgs),
617 ],
618 ),
619 content_snapshot=snapshot(external("ae070a447e1d*.bin")),
620 mock_client=client,
621 respx_mock=respx_mock,
622 )
623
624 assert print_obj(listener.stream.current_completion_snapshot.choices, monkeypatch) == snapshot(
625 """\
626[
627 ParsedChoice[object](
628 finish_reason='tool_calls',
629 index=0,
630 logprobs=None,
631 message=ParsedChatCompletionMessage[object](
632 content=None,
633 function_call=None,
634 parsed=None,
635 refusal=None,
636 role='assistant',
637 tool_calls=[
638 ParsedFunctionToolCall(
639 function=ParsedFunction(
640 arguments='{"city":"Edinburgh","country":"UK","units":"c"}',
641 name='GetWeatherArgs',
642 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='UK', units='c')
643 ),
644 id='call_Vz6ZXciy6Y0PYfT4d9W7fYB4',
645 index=0,
646 type='function'
647 )
648 ]
649 )
650 )
651]
652"""
653 )
654
655 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
656 """\
657[
658 ParsedChoice[NoneType](
659 finish_reason='tool_calls',
660 index=0,
661 logprobs=None,
662 message=ParsedChatCompletionMessage[NoneType](
663 content=None,
664 function_call=None,
665 parsed=None,
666 refusal=None,
667 role='assistant',
668 tool_calls=[
669 ParsedFunctionToolCall(
670 function=ParsedFunction(
671 arguments='{"city":"Edinburgh","country":"UK","units":"c"}',
672 name='GetWeatherArgs',
673 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='UK', units='c')
674 ),
675 id='call_Vz6ZXciy6Y0PYfT4d9W7fYB4',
676 index=0,
677 type='function'
678 )
679 ]
680 )
681 )
682]
683"""
684 )
685
686
687@pytest.mark.respx(base_url=base_url)
688def test_parse_multiple_pydantic_tools(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
689 class GetWeatherArgs(BaseModel):
690 """Get the temperature for the given country/city combo"""
691
692 city: str
693 country: str
694 units: Literal["c", "f"] = "c"
695
696 class GetStockPrice(BaseModel):
697 ticker: str
698 exchange: str
699
700 listener = _make_stream_snapshot_request(
701 lambda c: c.beta.chat.completions.stream(
702 model="gpt-4o-2024-08-06",
703 messages=[
704 {
705 "role": "user",
706 "content": "What's the weather like in Edinburgh?",
707 },
708 {
709 "role": "user",
710 "content": "What's the price of AAPL?",
711 },
712 ],
713 tools=[
714 openai.pydantic_function_tool(GetWeatherArgs),
715 openai.pydantic_function_tool(
716 GetStockPrice, name="get_stock_price", description="Fetch the latest price for a given ticker"
717 ),
718 ],
719 ),
720 content_snapshot=snapshot(external("a346213bec7a*.bin")),
721 mock_client=client,
722 respx_mock=respx_mock,
723 )
724
725 assert print_obj(listener.stream.current_completion_snapshot.choices, monkeypatch) == snapshot(
726 """\
727[
728 ParsedChoice[object](
729 finish_reason='tool_calls',
730 index=0,
731 logprobs=None,
732 message=ParsedChatCompletionMessage[object](
733 content=None,
734 function_call=None,
735 parsed=None,
736 refusal=None,
737 role='assistant',
738 tool_calls=[
739 ParsedFunctionToolCall(
740 function=ParsedFunction(
741 arguments='{"city": "Edinburgh", "country": "UK", "units": "c"}',
742 name='GetWeatherArgs',
743 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='UK', units='c')
744 ),
745 id='call_g4Q1vRbE0CaHGOs5if8mHsBq',
746 index=0,
747 type='function'
748 ),
749 ParsedFunctionToolCall(
750 function=ParsedFunction(
751 arguments='{"ticker": "AAPL", "exchange": "NASDAQ"}',
752 name='get_stock_price',
753 parsed_arguments=GetStockPrice(exchange='NASDAQ', ticker='AAPL')
754 ),
755 id='call_gWj3HQxZEHnFvyJLEHIiJKBV',
756 index=1,
757 type='function'
758 )
759 ]
760 )
761 )
762]
763"""
764 )
765 completion = listener.stream.get_final_completion()
766 assert print_obj(completion.choices[0].message.tool_calls, monkeypatch) == snapshot(
767 """\
768[
769 ParsedFunctionToolCall(
770 function=ParsedFunction(
771 arguments='{"city": "Edinburgh", "country": "UK", "units": "c"}',
772 name='GetWeatherArgs',
773 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='UK', units='c')
774 ),
775 id='call_g4Q1vRbE0CaHGOs5if8mHsBq',
776 index=0,
777 type='function'
778 ),
779 ParsedFunctionToolCall(
780 function=ParsedFunction(
781 arguments='{"ticker": "AAPL", "exchange": "NASDAQ"}',
782 name='get_stock_price',
783 parsed_arguments=GetStockPrice(exchange='NASDAQ', ticker='AAPL')
784 ),
785 id='call_gWj3HQxZEHnFvyJLEHIiJKBV',
786 index=1,
787 type='function'
788 )
789]
790"""
791 )
792
793
794@pytest.mark.respx(base_url=base_url)
795def test_parse_strict_tools(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
796 listener = _make_stream_snapshot_request(
797 lambda c: c.beta.chat.completions.stream(
798 model="gpt-4o-2024-08-06",
799 messages=[
800 {
801 "role": "user",
802 "content": "What's the weather like in SF?",
803 },
804 ],
805 tools=[
806 {
807 "type": "function",
808 "function": {
809 "name": "get_weather",
810 "parameters": {
811 "type": "object",
812 "properties": {
813 "city": {"type": "string"},
814 "state": {"type": "string"},
815 },
816 "required": [
817 "city",
818 "state",
819 ],
820 "additionalProperties": False,
821 },
822 "strict": True,
823 },
824 }
825 ],
826 ),
827 content_snapshot=snapshot(external("a7097cae6a1f*.bin")),
828 mock_client=client,
829 respx_mock=respx_mock,
830 )
831
832 assert print_obj(listener.stream.current_completion_snapshot.choices, monkeypatch) == snapshot(
833 """\
834[
835 ParsedChoice[object](
836 finish_reason='tool_calls',
837 index=0,
838 logprobs=None,
839 message=ParsedChatCompletionMessage[object](
840 content=None,
841 function_call=None,
842 parsed=None,
843 refusal=None,
844 role='assistant',
845 tool_calls=[
846 ParsedFunctionToolCall(
847 function=ParsedFunction(
848 arguments='{"city":"San Francisco","state":"CA"}',
849 name='get_weather',
850 parsed_arguments={'city': 'San Francisco', 'state': 'CA'}
851 ),
852 id='call_rQe3kzGnTr2epjx8HREg3F2a',
853 index=0,
854 type='function'
855 )
856 ]
857 )
858 )
859]
860"""
861 )
862
863
864@pytest.mark.respx(base_url=base_url)
865def test_non_pydantic_response_format(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
866 listener = _make_stream_snapshot_request(
867 lambda c: c.beta.chat.completions.stream(
868 model="gpt-4o-2024-08-06",
869 messages=[
870 {
871 "role": "user",
872 "content": "What's the weather like in SF? Give me any JSON back",
873 },
874 ],
875 response_format={"type": "json_object"},
876 ),
877 content_snapshot=snapshot(external("3e0df46f250d*.bin")),
878 mock_client=client,
879 respx_mock=respx_mock,
880 )
881
882 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
883 """\
884[
885 ParsedChoice[NoneType](
886 finish_reason='stop',
887 index=0,
888 logprobs=None,
889 message=ParsedChatCompletionMessage[NoneType](
890 content='{\\n "location": "San Francisco, CA",\\n "temperature": "N/A",\\n "conditions": "N/A",\\n
891"humidity": "N/A",\\n "wind_speed": "N/A",\\n "timestamp": "N/A",\\n "note": "Real-time weather data is not available.
892Please check a reliable weather service for the most up-to-date information on San Francisco\\'s weather conditions."}',
893 function_call=None,
894 parsed=None,
895 refusal=None,
896 role='assistant',
897 tool_calls=[]
898 )
899 )
900]
901"""
902 )
903
904
905@pytest.mark.respx(base_url=base_url)
906def test_allows_non_strict_tools_but_no_parsing(
907 client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch
908) -> None:
909 listener = _make_stream_snapshot_request(
910 lambda c: c.beta.chat.completions.stream(
911 model="gpt-4o-2024-08-06",
912 messages=[{"role": "user", "content": "what's the weather in NYC?"}],
913 tools=[
914 {
915 "type": "function",
916 "function": {
917 "name": "get_weather",
918 "parameters": {"type": "object", "properties": {"city": {"type": "string"}}},
919 },
920 }
921 ],
922 ),
923 content_snapshot=snapshot(external("fb75060ede89*.bin")),
924 mock_client=client,
925 respx_mock=respx_mock,
926 )
927
928 assert print_obj(listener.get_event_by_type("tool_calls.function.arguments.done"), monkeypatch) == snapshot("""\
929FunctionToolCallArgumentsDoneEvent(
930 arguments='{"city":"New York City"}',
931 index=0,
932 name='get_weather',
933 parsed_arguments=None,
934 type='tool_calls.function.arguments.done'
935)
936""")
937
938 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
939 """\
940[
941 ParsedChoice[NoneType](
942 finish_reason='stop',
943 index=0,
944 logprobs=None,
945 message=ParsedChatCompletionMessage[NoneType](
946 content=None,
947 function_call=None,
948 parsed=None,
949 refusal=None,
950 role='assistant',
951 tool_calls=[
952 ParsedFunctionToolCall(
953 function=ParsedFunction(
954 arguments='{"city":"New York City"}',
955 name='get_weather',
956 parsed_arguments=None
957 ),
958 id='call_9rqjEc1DQRADTYGVV45LbZwL',
959 index=0,
960 type='function'
961 )
962 ]
963 )
964 )
965]
966"""
967 )
968
969
970@pytest.mark.parametrize("sync", [True, False], ids=["sync", "async"])
971def test_stream_method_in_sync(sync: bool, client: OpenAI, async_client: AsyncOpenAI) -> None:
972 checking_client: OpenAI | AsyncOpenAI = client if sync else async_client
973
974 assert_signatures_in_sync(
975 checking_client.chat.completions.create,
976 checking_client.beta.chat.completions.stream,
977 exclude_params={"response_format", "stream"},
978 )
979
980
981class StreamListener(Generic[ResponseFormatT]):
982 def __init__(self, stream: ChatCompletionStream[ResponseFormatT]) -> None:
983 self.stream = stream
984 self.events: list[ChatCompletionStreamEvent[ResponseFormatT]] = []
985
986 def __iter__(self) -> Iterator[ChatCompletionStreamEvent[ResponseFormatT]]:
987 for event in self.stream:
988 self.events.append(event)
989 yield event
990
991 @overload
992 def get_event_by_type(self, event_type: Literal["content.done"]) -> ContentDoneEvent[ResponseFormatT] | None: ...
993
994 @overload
995 def get_event_by_type(self, event_type: str) -> ChatCompletionStreamEvent[ResponseFormatT] | None: ...
996
997 def get_event_by_type(self, event_type: str) -> ChatCompletionStreamEvent[ResponseFormatT] | None:
998 return next((e for e in self.events if e.type == event_type), None)
999
1000
1001def _make_stream_snapshot_request(
1002 func: Callable[[OpenAI], ChatCompletionStreamManager[ResponseFormatT]],
1003 *,
1004 content_snapshot: Any,
1005 respx_mock: MockRouter,
1006 mock_client: OpenAI,
1007 on_event: Callable[[ChatCompletionStream[ResponseFormatT], ChatCompletionStreamEvent[ResponseFormatT]], Any]
1008 | None = None,
1009) -> StreamListener[ResponseFormatT]:
1010 live = os.environ.get("OPENAI_LIVE") == "1"
1011 if live:
1012
1013 def _on_response(response: httpx.Response) -> None:
1014 # update the content snapshot
1015 assert outsource(response.read()) == content_snapshot
1016
1017 respx_mock.stop()
1018
1019 client = OpenAI(
1020 http_client=httpx.Client(
1021 event_hooks={
1022 "response": [_on_response],
1023 }
1024 )
1025 )
1026 else:
1027 respx_mock.post("/chat/completions").mock(
1028 return_value=httpx.Response(
1029 200,
1030 content=content_snapshot._old_value._load_value(),
1031 headers={"content-type": "text/event-stream"},
1032 )
1033 )
1034
1035 client = mock_client
1036
1037 with func(client) as stream:
1038 listener = StreamListener(stream)
1039
1040 for event in listener:
1041 if on_event:
1042 on_event(stream, event)
1043
1044 if live:
1045 client.close()
1046
1047 return listener
1048