openai/tiktoken

Public

main

Find a branch or tag

HTTPS

tests/test_helpers.py

22lines · modecode

3 years ago

1	`import bisect`
2	`import functools`
3	`import os`
4
5	`import pytest`
6
7	`import tiktoken`
8
9	`MAX_EXAMPLES: int = int(os.environ.get("TIKTOKEN_MAX_EXAMPLES", "100"))`
10
11	`ENCODINGS = ["r50k_base", "cl100k_base"]`
12	`SOME_ENCODINGS = ["cl100k_base"]`
13
14
15	`ENCODING_FACTORIES = [`
16	`pytest.param(functools.partial(tiktoken.get_encoding, name), id=name) for name in ENCODINGS`
17	`]`
18	`SOME_ENCODING_FACTORIES = [`
19	`pytest.param(functools.partial(tiktoken.get_encoding, name), id=name) for name in SOME_ENCODINGS`
20	`]`