openai/tiktoken
Publicmirrored from https://github.com/openai/tiktokenAvailable
pyproject.toml
45lines · modecode
| 1 | [project] |
| 2 | name = "tiktoken" |
| 3 | version = "0.6.0" |
| 4 | description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models" |
| 5 | readme = "README.md" |
| 6 | license = {file = "LICENSE"} |
| 7 | authors = [{name = "Shantanu Jain"}, {email = "shantanu@openai.com"}] |
| 8 | dependencies = ["regex>=2022.1.18", "requests>=2.26.0"] |
| 9 | optional-dependencies = {blobfile = ["blobfile>=2"]} |
| 10 | requires-python = ">=3.8" |
| 11 | |
| 12 | [project.urls] |
| 13 | homepage = "https://github.com/openai/tiktoken" |
| 14 | repository = "https://github.com/openai/tiktoken" |
| 15 | changelog = "https://github.com/openai/tiktoken/blob/main/CHANGELOG.md" |
| 16 | |
| 17 | [build-system] |
| 18 | build-backend = "setuptools.build_meta" |
| 19 | requires = ["setuptools>=62.4", "wheel", "setuptools-rust>=1.5.2"] |
| 20 | |
| 21 | [tool.cibuildwheel] |
| 22 | build-frontend = "build" |
| 23 | build-verbosity = 1 |
| 24 | |
| 25 | linux.before-all = "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y" |
| 26 | linux.environment = { PATH = "$PATH:$HOME/.cargo/bin" } |
| 27 | macos.before-all = "rustup target add aarch64-apple-darwin" |
| 28 | |
| 29 | skip = [ |
| 30 | "*-manylinux_i686", |
| 31 | "*-musllinux_i686", |
| 32 | "*-win32", |
| 33 | ] |
| 34 | macos.archs = ["x86_64", "arm64"] |
| 35 | # When cross-compiling on Intel, it is not possible to test arm64 wheels. |
| 36 | # Warnings will be silenced with following CIBW_TEST_SKIP |
| 37 | test-skip = "*-macosx_arm64" |
| 38 | |
| 39 | before-test = "pip install pytest hypothesis" |
| 40 | test-command = "pytest {project}/tests --import-mode=append" |
| 41 | |
| 42 | [[tool.cibuildwheel.overrides]] |
| 43 | select = "*linux_aarch64" |
| 44 | test-command = """python -c 'import tiktoken; enc = tiktoken.get_encoding("gpt2"); assert enc.encode("hello world") == [31373, 995]'""" |
| 45 | |
| 46 | |