Remove dependency on requests

Author: tristan-jl

pyproject.toml

@@ -5,7 +5,7 @@ description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models"
 readme = "README.md"
 license = {file = "LICENSE"}
 authors = [{name = "Shantanu Jain"}, {email = "[email protected]"}]
-dependencies = ["regex>=2022.1.18", "requests>=2.26.0"]
+dependencies = ["regex>=2022.1.18"]
 optional-dependencies = {blobfile = ["blobfile>=2"]}
 requires-python = ">=3.8"
 

tiktoken/load.py

@@ -8,7 +8,7 @@
 import uuid
 from typing import Optional
 
-import requests
+import urllib.request
 
 
 def read_file(blobpath: str) -> bytes:
@@ -22,9 +22,9 @@ def read_file(blobpath: str) -> bytes:
         with blobfile.BlobFile(blobpath, "rb") as f:
             return f.read()
     # avoiding blobfile for public files helps avoid auth issues, like MFA prompts
-    resp = requests.get(blobpath)
-    resp.raise_for_status()
-    return resp.content
+    with urllib.request.urlopen(blobpath) as response:
+        resp = response.read()
+    return resp
 
 
 def check_hash(data: bytes, expected_hash: str) -> bool: