openai/tiktoken

Public

main

Find a branch or tag

HTTPS

tests/test_misc.py

30lines · modecode

unknown

1	`import subprocess`
2	`import sys`
3
4	`import tiktoken`
5
6
7	`def test_encoding_for_model():`
8	`enc = tiktoken.encoding_for_model("gpt2")`
9	`assert enc.name == "gpt2"`
10	`enc = tiktoken.encoding_for_model("text-davinci-003")`
11	`assert enc.name == "p50k_base"`
12	`enc = tiktoken.encoding_for_model("text-davinci-edit-001")`
13	`assert enc.name == "p50k_edit"`
14	`enc = tiktoken.encoding_for_model("gpt-3.5-turbo-0301")`
15	`assert enc.name == "cl100k_base"`
16	`enc = tiktoken.encoding_for_model("gpt-4")`
17	`assert enc.name == "cl100k_base"`
18	`enc = tiktoken.encoding_for_model("gpt-4o")`
19	`assert enc.name == "o200k_base"`
20	`enc = tiktoken.encoding_for_model("gpt-oss-120b")`
21	`assert enc.name == "o200k_harmony"`
22
23
24	`def test_optional_blobfile_dependency():`
25	`prog = """`
26	`import tiktoken`
27	`import sys`
28	`assert "blobfile" not in sys.modules`
29	`"""`
30	`subprocess.check_call([sys.executable, "-c", prog])`
31