Partial sync of codebase

Author: hauntsaninja

Cargo.toml

@@ -5,11 +5,20 @@ edition = "2021"
 rust-version = "1.57.0"
 
 [lib]
-name = "_tiktoken"
-crate-type = ["cdylib"]
+name = "tiktoken"
+crate-type = ["cdylib", "rlib"]
+
+[features]
+default = []
+python = [
+    "pyo3",
+]
 
 [dependencies]
-pyo3 = { version = "0.22.2", default-features = false, features = ["extension-module", "macros"] }
+pyo3 = { version = "0.22.2", default-features = false, features = [
+    "extension-module",
+    "macros",
+], optional = true }
 
 # tiktoken dependencies
 fancy-regex = "0.13.0"

pyproject.toml

@@ -3,10 +3,10 @@ name = "tiktoken"
 version = "0.8.0"
 description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models"
 readme = "README.md"
-license = {file = "LICENSE"}
-authors = [{name = "Shantanu Jain"}, {email = "[email protected]"}]
+license = { file = "LICENSE" }
+authors = [{ name = "Shantanu Jain" }, { email = "[email protected]" }]
 dependencies = ["regex>=2022.1.18", "requests>=2.26.0"]
-optional-dependencies = {blobfile = ["blobfile>=2"]}
+optional-dependencies = { blobfile = ["blobfile>=2"] }
 requires-python = ">=3.9"
 
 [project.urls]
@@ -43,3 +43,4 @@ test-command = "pytest {project}/tests --import-mode=append"
 [[tool.cibuildwheel.overrides]]
 select = "*linux_aarch64"
 test-command = """python -c 'import tiktoken; enc = tiktoken.get_encoding("gpt2"); assert enc.encode("hello world") == [31373, 995]'"""
+

setup.py

@@ -10,6 +10,7 @@
             # Between our use of editable installs and wanting to use Rust for performance sensitive
             # code, it makes sense to just always use --release
             debug=False,
+            features=["python"],
         )
     ],
     package_data={"tiktoken": ["py.typed"]},