Refine mt5 (#410)

* refine global_set of position_bias * reformat
Oneflow-Inc · Oct 28, 2022 · b3c5ba2 · b3c5ba2
1 parent 9a4af26
commit b3c5ba2
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/projects/MT5/layers/attention_layer.py b/projects/MT5/layers/attention_layer.py
@@ -217,8 +217,8 @@ def forward(
                 position_bias = position_bias[:, :, -hidden_states.size(1) :, :]
 
             position_bias = position_bias + (1 - attention_mask) * -1000
+            position_bias = position_bias.to_global(placement=attention_scores.placement)
 
-        position_bias = position_bias.to_global(placement=attention_scores.placement)
         attention_scores = attention_scores + position_bias
 
         if attention_mask is not None:

diff --git a/projects/T5/models/attention.py b/projects/T5/models/attention.py
@@ -224,8 +224,8 @@ def forward(
                 position_bias = position_bias[:, :, -hidden_states.size(1) :, :]
 
             position_bias = position_bias + (1 - attention_mask) * -1000
+            position_bias = position_bias.to_global(placement=attention_scores.placement)
 
-        position_bias = position_bias.to_global(placement=attention_scores.placement)
         attention_scores = attention_scores + position_bias
 
         # [S(0), S(1)] x [S(0), B] = [S(0), S(1)]