automatically load dtype from config (#698)

casper-hansen · Jan 20, 2025 · 9affc3e · 9affc3e
1 parent 578b8c9
commit 9affc3e
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/awq/models/auto.py b/awq/models/auto.py
@@ -64,7 +64,7 @@ def __init__(self):
     def from_pretrained(
         self,
         model_path,
-        torch_dtype=torch.float16,
+        torch_dtype="auto",
         trust_remote_code=True,
         safetensors=True,
         device_map=None,