openai/tiktoken
Publicmirrored fromhttps://github.com/openai/tiktokenAvailable
Cargo.toml
26lines · modecode
| 1 | [package] |
| 2 | name = "tiktoken" |
| 3 | version = "0.11.0" |
| 4 | edition = "2024" |
| 5 | |
| 6 | [lib] |
| 7 | name = "tiktoken" |
| 8 | crate-type = ["cdylib", "rlib"] |
| 9 | |
| 10 | [features] |
| 11 | default = [] |
| 12 | python = [ |
| 13 | "pyo3", |
| 14 | ] |
| 15 | |
| 16 | [dependencies] |
| 17 | pyo3 = { version = "0.24.1", default-features = false, features = [ |
| 18 | "extension-module", |
| 19 | "macros", |
| 20 | ], optional = true } |
| 21 | |
| 22 | # tiktoken dependencies |
| 23 | fancy-regex = "0.13.0" |
| 24 | regex = "1.10.3" |
| 25 | rustc-hash = "1.1.0" |
| 26 | bstr = "1.5.0" |
| 27 | |