openai/tiktoken
Publicmirrored from https://github.com/openai/tiktokenAvailable
tiktoken_ext/openai_public.py
88lines · modecode
| 1 | from tiktoken.load import data_gym_to_mergeable_bpe_ranks, load_tiktoken_bpe |
| 2 | |
| 3 | ENDOFTEXT = "<|endoftext|>" |
| 4 | FIM_PREFIX = "<|fim_prefix|>" |
| 5 | FIM_MIDDLE = "<|fim_middle|>" |
| 6 | FIM_SUFFIX = "<|fim_suffix|>" |
| 7 | ENDOFPROMPT = "<|endofprompt|>" |
| 8 | |
| 9 | |
| 10 | def gpt2(): |
| 11 | mergeable_ranks = data_gym_to_mergeable_bpe_ranks( |
| 12 | vocab_bpe_file="https://openaipublic.blob.core.windows.net/gpt-2/encodings/main/vocab.bpe", |
| 13 | encoder_json_file="https://openaipublic.blob.core.windows.net/gpt-2/encodings/main/encoder.json", |
| 14 | ) |
| 15 | return { |
| 16 | "name": "gpt2", |
| 17 | "explicit_n_vocab": 50257, |
| 18 | "pat_str": r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""", |
| 19 | "mergeable_ranks": mergeable_ranks, |
| 20 | "special_tokens": {ENDOFTEXT: 50256}, |
| 21 | } |
| 22 | |
| 23 | |
| 24 | def r50k_base(): |
| 25 | mergeable_ranks = load_tiktoken_bpe( |
| 26 | "https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken" |
| 27 | ) |
| 28 | return { |
| 29 | "name": "r50k_base", |
| 30 | "explicit_n_vocab": 50257, |
| 31 | "pat_str": r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""", |
| 32 | "mergeable_ranks": mergeable_ranks, |
| 33 | "special_tokens": {ENDOFTEXT: 50256}, |
| 34 | } |
| 35 | |
| 36 | |
| 37 | def p50k_base(): |
| 38 | mergeable_ranks = load_tiktoken_bpe( |
| 39 | "https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken" |
| 40 | ) |
| 41 | return { |
| 42 | "name": "p50k_base", |
| 43 | "explicit_n_vocab": 50281, |
| 44 | "pat_str": r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""", |
| 45 | "mergeable_ranks": mergeable_ranks, |
| 46 | "special_tokens": {ENDOFTEXT: 50256}, |
| 47 | } |
| 48 | |
| 49 | |
| 50 | def p50k_edit(): |
| 51 | mergeable_ranks = load_tiktoken_bpe( |
| 52 | "https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken" |
| 53 | ) |
| 54 | special_tokens = {ENDOFTEXT: 50256, FIM_PREFIX: 50281, FIM_MIDDLE: 50282, FIM_SUFFIX: 50283} |
| 55 | return { |
| 56 | "name": "p50k_edit", |
| 57 | "pat_str": r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""", |
| 58 | "mergeable_ranks": mergeable_ranks, |
| 59 | "special_tokens": special_tokens, |
| 60 | } |
| 61 | |
| 62 | |
| 63 | def cl100k_base(): |
| 64 | mergeable_ranks = load_tiktoken_bpe( |
| 65 | "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken" |
| 66 | ) |
| 67 | special_tokens = { |
| 68 | ENDOFTEXT: 100257, |
| 69 | FIM_PREFIX: 100258, |
| 70 | FIM_MIDDLE: 100259, |
| 71 | FIM_SUFFIX: 100260, |
| 72 | ENDOFPROMPT: 100276, |
| 73 | } |
| 74 | return { |
| 75 | "name": "cl100k_base", |
| 76 | "pat_str": r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+""", |
| 77 | "mergeable_ranks": mergeable_ranks, |
| 78 | "special_tokens": special_tokens, |
| 79 | } |
| 80 | |
| 81 | |
| 82 | ENCODING_CONSTRUCTORS = { |
| 83 | "gpt2": gpt2, |
| 84 | "r50k_base": r50k_base, |
| 85 | "p50k_base": p50k_base, |
| 86 | "p50k_edit": p50k_edit, |
| 87 | "cl100k_base": cl100k_base, |
| 88 | } |
| 89 | |