openai/tiktoken
Publicmirrored from https://github.com/openai/tiktokenAvailable
tests/test_encoding.py
256lines · modecode
| 1 | # Note that there are more actual tests, they're just not currently public :-) |
| 2 | |
| 3 | from typing import Callable |
| 4 | |
| 5 | import hypothesis |
| 6 | import hypothesis.strategies as st |
| 7 | import pytest |
| 8 | |
| 9 | import tiktoken |
| 10 | |
| 11 | from .test_helpers import ENCODING_FACTORIES, MAX_EXAMPLES |
| 12 | |
| 13 | |
| 14 | def test_simple(): |
| 15 | enc = tiktoken.get_encoding("gpt2") |
| 16 | assert enc.encode("hello world") == [31373, 995] |
| 17 | assert enc.decode([31373, 995]) == "hello world" |
| 18 | assert enc.encode("hello <|endoftext|>", allowed_special="all") == [31373, 220, 50256] |
| 19 | |
| 20 | enc = tiktoken.get_encoding("cl100k_base") |
| 21 | assert enc.encode("hello world") == [15339, 1917] |
| 22 | assert enc.decode([15339, 1917]) == "hello world" |
| 23 | assert enc.encode("hello <|endoftext|>", allowed_special="all") == [15339, 220, 100257] |
| 24 | |
| 25 | for enc_name in tiktoken.list_encoding_names(): |
| 26 | enc = tiktoken.get_encoding(enc_name) |
| 27 | for token in range(min(10_000, enc.max_token_value - 1)): |
| 28 | assert enc.encode_single_token(enc.decode_single_token_bytes(token)) == token |
| 29 | |
| 30 | |
| 31 | def test_simple_repeated(): |
| 32 | enc = tiktoken.get_encoding("gpt2") |
| 33 | assert enc.encode("0") == [15] |
| 34 | assert enc.encode("00") == [405] |
| 35 | assert enc.encode("000") == [830] |
| 36 | assert enc.encode("0000") == [2388] |
| 37 | assert enc.encode("00000") == [20483] |
| 38 | assert enc.encode("000000") == [10535] |
| 39 | assert enc.encode("0000000") == [24598] |
| 40 | assert enc.encode("00000000") == [8269] |
| 41 | assert enc.encode("000000000") == [10535, 830] |
| 42 | assert enc.encode("0000000000") == [8269, 405] |
| 43 | assert enc.encode("00000000000") == [8269, 830] |
| 44 | assert enc.encode("000000000000") == [8269, 2388] |
| 45 | assert enc.encode("0000000000000") == [8269, 20483] |
| 46 | assert enc.encode("00000000000000") == [8269, 10535] |
| 47 | assert enc.encode("000000000000000") == [8269, 24598] |
| 48 | assert enc.encode("0000000000000000") == [25645] |
| 49 | assert enc.encode("00000000000000000") == [8269, 10535, 830] |
| 50 | |
| 51 | |
| 52 | def test_simple_regex(): |
| 53 | enc = tiktoken.get_encoding("cl100k_base") |
| 54 | assert enc.encode("rer") == [38149] |
| 55 | assert enc.encode("'rer") == [2351, 81] |
| 56 | assert enc.encode("today\n ") == [31213, 198, 220] |
| 57 | assert enc.encode("today\n \n") == [31213, 27907] |
| 58 | assert enc.encode("today\n \n") == [31213, 14211] |
| 59 | |
| 60 | |
| 61 | def test_basic_encode(): |
| 62 | enc = tiktoken.get_encoding("r50k_base") |
| 63 | assert enc.encode("hello world") == [31373, 995] |
| 64 | |
| 65 | enc = tiktoken.get_encoding("p50k_base") |
| 66 | assert enc.encode("hello world") == [31373, 995] |
| 67 | |
| 68 | enc = tiktoken.get_encoding("cl100k_base") |
| 69 | assert enc.encode("hello world") == [15339, 1917] |
| 70 | assert enc.encode(" \x850") == [220, 126, 227, 15] |
| 71 | |
| 72 | |
| 73 | def test_encode_empty(): |
| 74 | enc = tiktoken.get_encoding("r50k_base") |
| 75 | assert enc.encode("") == [] |
| 76 | |
| 77 | |
| 78 | def test_encode_bytes(): |
| 79 | enc = tiktoken.get_encoding("cl100k_base") |
| 80 | assert enc._encode_bytes(b" \xec\x8b\xa4\xed") == [62085] |
| 81 | for i in range(10): |
| 82 | bytestring = b"\x80" * i |
| 83 | assert enc.decode_bytes(enc._encode_bytes(bytestring)) == bytestring |
| 84 | |
| 85 | |
| 86 | @pytest.mark.parametrize("make_enc", ENCODING_FACTORIES) |
| 87 | @hypothesis.given(bytestring=st.binary()) |
| 88 | @hypothesis.settings(deadline=None) |
| 89 | def test_hyp_encode_bytes(make_enc: Callable[[], tiktoken.Encoding], bytestring: bytes): |
| 90 | enc = make_enc() |
| 91 | assert enc.decode_bytes(enc._encode_bytes(bytestring)) == bytestring |
| 92 | |
| 93 | |
| 94 | def test_encode_surrogate_pairs(): |
| 95 | enc = tiktoken.get_encoding("cl100k_base") |
| 96 | |
| 97 | assert enc.encode("👍") == [9468, 239, 235] |
| 98 | # surrogate pair gets converted to codepoint |
| 99 | assert enc.encode("\ud83d\udc4d") == [9468, 239, 235] |
| 100 | |
| 101 | # lone surrogate just gets replaced |
| 102 | assert enc.encode("\ud83d") == enc.encode("�") |
| 103 | |
| 104 | |
| 105 | @pytest.mark.parametrize("make_enc", ENCODING_FACTORIES) |
| 106 | def test_catastrophically_repetitive(make_enc: Callable[[], tiktoken.Encoding]): |
| 107 | enc = make_enc() |
| 108 | for c in ["^", "0", "a", "'s", " ", "\n"]: |
| 109 | big_value = c * 10_000 |
| 110 | assert big_value == enc.decode(enc.encode(big_value)) |
| 111 | |
| 112 | big_value = " " + big_value |
| 113 | assert big_value == enc.decode(enc.encode(big_value)) |
| 114 | |
| 115 | big_value = big_value + "\n" |
| 116 | assert big_value == enc.decode(enc.encode(big_value)) |
| 117 | |
| 118 | |
| 119 | # ==================== |
| 120 | # Roundtrip |
| 121 | # ==================== |
| 122 | |
| 123 | |
| 124 | @pytest.mark.parametrize("make_enc", ENCODING_FACTORIES) |
| 125 | def test_basic_roundtrip(make_enc): |
| 126 | enc = make_enc() |
| 127 | for value in ( |
| 128 | "hello", |
| 129 | "hello ", |
| 130 | "hello ", |
| 131 | " hello", |
| 132 | " hello ", |
| 133 | " hello ", |
| 134 | "hello world", |
| 135 | "请考试我的软件！12345", |
| 136 | ): |
| 137 | assert value == enc.decode(enc.encode(value)) |
| 138 | assert value == enc.decode(enc.encode_ordinary(value)) |
| 139 | |
| 140 | |
| 141 | @pytest.mark.parametrize("make_enc", ENCODING_FACTORIES) |
| 142 | @hypothesis.given(text=st.text()) |
| 143 | @hypothesis.settings(deadline=None) |
| 144 | def test_hyp_roundtrip(make_enc: Callable[[], tiktoken.Encoding], text): |
| 145 | enc = make_enc() |
| 146 | |
| 147 | assert text == enc.decode(enc.encode(text)) |
| 148 | |
| 149 | |
| 150 | @pytest.mark.parametrize("make_enc", ENCODING_FACTORIES) |
| 151 | def test_single_token_roundtrip(make_enc: Callable[[], tiktoken.Encoding]): |
| 152 | enc = make_enc() |
| 153 | |
| 154 | for token in range(enc.n_vocab): |
| 155 | try: |
| 156 | token_bytes = enc.decode_single_token_bytes(token) |
| 157 | except KeyError: |
| 158 | continue |
| 159 | assert enc.encode_single_token(token_bytes) == token |
| 160 | |
| 161 | |
| 162 | # ==================== |
| 163 | # Special tokens |
| 164 | # ==================== |
| 165 | |
| 166 | |
| 167 | def test_special_token(): |
| 168 | enc = tiktoken.get_encoding("cl100k_base") |
| 169 | |
| 170 | eot = enc.encode_single_token("<|endoftext|>") |
| 171 | assert eot == enc.eot_token |
| 172 | fip = enc.encode_single_token("<|fim_prefix|>") |
| 173 | fim = enc.encode_single_token("<|fim_middle|>") |
| 174 | |
| 175 | text = "<|endoftext|> hello <|fim_prefix|>" |
| 176 | assert eot not in enc.encode(text, disallowed_special=()) |
| 177 | with pytest.raises(ValueError): |
| 178 | enc.encode(text) |
| 179 | with pytest.raises(ValueError): |
| 180 | enc.encode(text, disallowed_special="all") |
| 181 | with pytest.raises(ValueError): |
| 182 | enc.encode(text, disallowed_special={"<|endoftext|>"}) |
| 183 | with pytest.raises(ValueError): |
| 184 | enc.encode(text, disallowed_special={"<|fim_prefix|>"}) |
| 185 | |
| 186 | text = "<|endoftext|> hello <|fim_prefix|> there <|fim_middle|>" |
| 187 | tokens = enc.encode(text, disallowed_special=()) |
| 188 | assert eot not in tokens |
| 189 | assert fip not in tokens |
| 190 | assert fim not in tokens |
| 191 | |
| 192 | tokens = enc.encode(text, allowed_special="all", disallowed_special=()) |
| 193 | assert eot in tokens |
| 194 | assert fip in tokens |
| 195 | assert fim in tokens |
| 196 | |
| 197 | tokens = enc.encode(text, allowed_special="all", disallowed_special="all") |
| 198 | assert eot in tokens |
| 199 | assert fip in tokens |
| 200 | assert fim in tokens |
| 201 | |
| 202 | tokens = enc.encode(text, allowed_special={"<|fim_prefix|>"}, disallowed_special=()) |
| 203 | assert eot not in tokens |
| 204 | assert fip in tokens |
| 205 | assert fim not in tokens |
| 206 | |
| 207 | tokens = enc.encode(text, allowed_special={"<|endoftext|>"}, disallowed_special=()) |
| 208 | assert eot in tokens |
| 209 | assert fip not in tokens |
| 210 | assert fim not in tokens |
| 211 | |
| 212 | tokens = enc.encode(text, allowed_special={"<|fim_middle|>"}, disallowed_special=()) |
| 213 | assert eot not in tokens |
| 214 | assert fip not in tokens |
| 215 | assert fim in tokens |
| 216 | |
| 217 | |
| 218 | @pytest.mark.parametrize("make_enc", ENCODING_FACTORIES) |
| 219 | @hypothesis.given(text=st.text()) |
| 220 | @hypothesis.settings(deadline=None, max_examples=MAX_EXAMPLES) |
| 221 | def test_hyp_special_ordinary(make_enc, text: str): |
| 222 | enc = make_enc() |
| 223 | assert enc.encode_ordinary(text) == enc.encode(text, disallowed_special=()) |
| 224 | |
| 225 | |
| 226 | # ==================== |
| 227 | # Batch encoding |
| 228 | # ==================== |
| 229 | |
| 230 | |
| 231 | @pytest.mark.parametrize("make_enc", ENCODING_FACTORIES) |
| 232 | def test_batch_encode(make_enc: Callable[[], tiktoken.Encoding]): |
| 233 | enc = make_enc() |
| 234 | text1 = "hello world" |
| 235 | text2 = "goodbye world" |
| 236 | |
| 237 | assert enc.encode_batch([text1]) == [enc.encode(text1)] |
| 238 | assert enc.encode_batch([text1, text2]) == [enc.encode(text1), enc.encode(text2)] |
| 239 | |
| 240 | assert enc.encode_ordinary_batch([text1]) == [enc.encode_ordinary(text1)] |
| 241 | assert enc.encode_ordinary_batch([text1, text2]) == [ |
| 242 | enc.encode_ordinary(text1), |
| 243 | enc.encode_ordinary(text2), |
| 244 | ] |
| 245 | |
| 246 | |
| 247 | @pytest.mark.parametrize("make_enc", ENCODING_FACTORIES) |
| 248 | @hypothesis.given(batch=st.lists(st.text())) |
| 249 | @hypothesis.settings(deadline=None) |
| 250 | def test_hyp_batch_roundtrip(make_enc: Callable[[], tiktoken.Encoding], batch): |
| 251 | enc = make_enc() |
| 252 | |
| 253 | encoded = enc.encode_batch(batch) |
| 254 | assert encoded == [enc.encode(t) for t in batch] |
| 255 | decoded = enc.decode_batch(encoded) |
| 256 | assert decoded == batch |
| 257 | |