diff --git a/QEfficient/base/modeling_qeff.py b/QEfficient/base/modeling_qeff.py
index cf53a8c70..1aafb1ba2 100644
--- a/QEfficient/base/modeling_qeff.py
+++ b/QEfficient/base/modeling_qeff.py
@@ -299,6 +299,8 @@ def _compile(
 
         if num_speculative_tokens:
             compile_hash.update(to_hashable({"num_speculative_tokens": num_speculative_tokens}))
+        # Hash num_devices too, since default value would always be 1.
+        compile_hash.update(to_hashable(mdp_ts_num_devices))
 
         # Check if already compiled
         compile_hash = compile_hash.hexdigest()[:16]