Add missing moves to torch autograd (pytorch#92772)

Applies some additional std::move functions to torch/csrc/autograd to opportunities that were found via static analysis.
Pull Request resolved: pytorch#92772
Approved by: https://github.com/ezyang

Author: Skylion007

torch/csrc/autograd/FunctionsManual.cpp

@@ -30,6 +30,7 @@
 #include <ciso646>
 #include <functional>
 #include <numeric>
+#include <utility>
 
 // Helper functions for autogenerated code
 // These used to be inlined into the codegened Functions.cpp
@@ -361,7 +362,7 @@ Tensor norm_jvp(
     const Tensor& self_t,
     const optional<Scalar>& p_,
     Tensor norm) {
-  return norm_jvp(self_p, self_t, p_, norm, {}, true);
+  return norm_jvp(self_p, self_t, p_, std::move(norm), {}, true);
 }
 
 Tensor _nested_from_padded_backward(
@@ -389,7 +390,7 @@ Tensor linalg_vector_norm_jvp(
   // No need to handle the dtype arg as it's handled via broadcasting in the
   // function
   auto dim = opt_dim.value_or(IntArrayRef({}));
-  return norm_jvp(self_p, self_t, scalar_ord, norm, dim, keepdim);
+  return norm_jvp(self_p, self_t, scalar_ord, std::move(norm), dim, keepdim);
 }
 
 Tensor linalg_vector_norm_backward(
@@ -402,7 +403,8 @@ Tensor linalg_vector_norm_backward(
   // No need to handle the dtype arg as it's handled via broadcasting in the
   // function
   auto dim = opt_dim.value_or(IntArrayRef({}));
-  return norm_backward(grad, self, scalar_ord, norm, dim, keepdim);
+  return norm_backward(
+      std::move(grad), self, scalar_ord, std::move(norm), dim, keepdim);
 }
 
 Tensor pow_backward(Tensor grad, const Tensor& self, const Scalar& exponent) {
@@ -415,7 +417,7 @@ Tensor pow_backward(Tensor grad, const Tensor& self, const Scalar& exponent) {
     Tensor out = (exponent.isComplex())
         ? grad_lambda(exponent.toComplexDouble())
         : grad_lambda(exponent.toDouble());
-    return handle_r_to_c(self, out);
+    return handle_r_to_c(self, std::move(out));
   }
 }
 
@@ -427,7 +429,7 @@ Tensor pow_backward_self(
       exponent == 0.0,
       at::zeros({}, grad.options()),
       grad * (exponent * self.pow(exponent - 1)).conj());
-  return handle_r_to_c(self, out);
+  return handle_r_to_c(self, std::move(out));
 }
 
 // Caveats:
@@ -455,7 +457,7 @@ Tensor pow_backward_exponent(
       grad *
       at::where(
           cond, at::zeros({}, grad.options()), (result * self.log()).conj());
-  return handle_r_to_c(exponent, out);
+  return handle_r_to_c(exponent, std::move(out));
 }
 
 Tensor pow_backward_exponent(
@@ -475,11 +477,11 @@ Tensor pow_backward_exponent(
     auto out = grad *
         at::where(cond(exponent),
                   at::zeros({}, grad.options()),
-                  grad_lambda(result, base));
-    return handle_r_to_c(exponent, out);
+                  grad_lambda(std::move(result), base));
+    return handle_r_to_c(exponent, std::move(out));
   } else {
-    auto out = grad * grad_lambda(result, base);
-    return handle_r_to_c(exponent, out);
+    auto out = grad * grad_lambda(std::move(result), base);
+    return handle_r_to_c(exponent, std::move(out));
   }
 }
 
@@ -521,7 +523,7 @@ Tensor masked_fill_backward(const Tensor& grad, const Tensor& mask) {
 
 Tensor mul_tensor_backward(Tensor grad, Tensor other, ScalarType self_st) {
   auto out = grad * other.conj();
-  return handle_r_to_c(self_st, out);
+  return handle_r_to_c(self_st, std::move(out));
 }
 
 Tensor div_tensor_self_backward(
@@ -534,11 +536,12 @@ Tensor div_tensor_self_backward(
   }
 
   auto result = grad / other.conj();
-  return handle_r_to_c(self_st, result);
+  return handle_r_to_c(self_st, std::move(result));
 }
 
 Tensor div_tensor_self_backward(Tensor grad, Tensor other, ScalarType self_st) {
-  return div_tensor_self_backward(grad, other, self_st, c10::nullopt);
+  return div_tensor_self_backward(
+      std::move(grad), std::move(other), self_st, c10::nullopt);
 }
 
 Tensor div_tensor_other_backward(
@@ -551,11 +554,12 @@ Tensor div_tensor_other_backward(
   }
 
   auto result = -grad * ((self / other) / other).conj();
-  return handle_r_to_c(other, result);
+  return handle_r_to_c(std::move(other), std::move(result));
 }
 
 Tensor div_tensor_other_backward(Tensor grad, Tensor self, Tensor other) {
-  return div_tensor_other_backward(grad, self, other, c10::nullopt);
+  return div_tensor_other_backward(
+      std::move(grad), std::move(self), std::move(other), c10::nullopt);
 }
 
 Tensor permute_backwards(const Tensor& grad, IntArrayRef fwd_dims) {
@@ -649,8 +653,9 @@ Tensor mean_backward(
     c10::SymInt numel,
     bool keepdim) {
   bool is_all_reduce = !opt_dim.has_value() || opt_dim.value().size() == 0;
-  auto n = is_all_reduce ? numel : _safe_size(shape, opt_dim.value());
-  return sum_backward(grad, shape, opt_dim, keepdim) / n;
+  auto n =
+      is_all_reduce ? std::move(numel) : _safe_size(shape, opt_dim.value());
+  return sum_backward(grad, shape, opt_dim, keepdim) / std::move(n);
 }
 
 std::vector<int64_t> reverse_list(const IntArrayRef list) {
@@ -692,7 +697,8 @@ Tensor prod_safe_zeros_backward(
 
   Tensor narrow_reverse =
       reverse_dim(inp.narrow(dim, 1, inp.size(dim) - 1), dim);
-  Tensor exclusive_reverse_nocp = at::cat({ones, narrow_reverse}, dim);
+  Tensor exclusive_reverse_nocp =
+      at::cat({std::move(ones), std::move(narrow_reverse)}, dim);
   Tensor exclusive_reverse =
       reverse_dim(exclusive_reverse_nocp.cumprod(dim), dim);
 
@@ -1387,8 +1393,8 @@ Tensor renorm_backward(
   }
   grad_output =
       grad_output.sum(reduce_dims, /*keepdim=*/true, /*dtype=*/real_acc_type);
-  auto nb =
-      norm_backward(grad_output, self, p, norm, reduce_dims, /*keepdim=*/true);
+  auto nb = norm_backward(
+      std::move(grad_output), self, p, norm, reduce_dims, /*keepdim=*/true);
 
   auto invnorm = (norm + 1e-7).reciprocal();
   auto grad_norm = maxnorm * invnorm * (grad - invnorm * nb);
@@ -1571,7 +1577,7 @@ Tensor std_backward(
     c10::optional<int64_t> correction,
     bool keepdim) {
   auto grad_var = (grad / (result * 2)).masked_fill_(result == 0, 0);
-  return var_backward(grad_var, self, dim, correction, keepdim);
+  return var_backward(std::move(grad_var), self, dim, correction, keepdim);
 }
 
 Tensor var_mean_backward(
@@ -1593,7 +1599,7 @@ Tensor var_mean_backward(
         dim_opt.value_or(IntArrayRef({})),
         self.sym_numel(),
         keepdim);
-    gself = gself.defined() ? gself + aux : aux;
+    gself = gself.defined() ? gself + aux : std::move(aux);
   }
   return gself;
 }
@@ -1618,7 +1624,7 @@ Tensor std_mean_backward(
         dim_opt.value_or(IntArrayRef({})),
         self.sym_numel(),
         keepdim);
-    gself = gself.defined() ? gself + aux : aux;
+    gself = gself.defined() ? gself + aux : std::move(aux);
   }
   return gself;
 }
@@ -1637,8 +1643,9 @@ Tensor masked_scatter_backward(
     // because mask_selected returns a 1-d tensor with size of masked elements
     // that are 1, we need to fill out the rest with zeros then reshape back to
     // tensor2's size.
-    auto zeros_fillin = at::zeros_symint({diff_nelem}, grad.options());
-    mask_selected = at::cat({mask_selected, zeros_fillin}, 0);
+    auto zeros_fillin =
+        at::zeros_symint({std::move(diff_nelem)}, grad.options());
+    mask_selected = at::cat({mask_selected, std::move(zeros_fillin)}, 0);
   }
   return mask_selected.view_symint(sizes);
 }
@@ -1661,7 +1668,7 @@ Tensor cholesky_jvp(const Tensor& dA, const Tensor& L, bool upper) {
   dL = at::linalg_solve_triangular(L_.mH(), dL, /*upper=*/true, /*left=*/false);
   dL = dL.tril() - dL.diagonal(0, -2, -1).mul(0.5).diag_embed();
   dL = L_.matmul(dL);
-  return upper ? dL.mH() : dL;
+  return upper ? dL.mH() : std::move(dL);
 }
 
 Tensor cholesky_backward(const Tensor& gL, bool upper, const Tensor& L) {
@@ -1899,7 +1906,7 @@ Tensor glu_double_backward(
   auto gI_second_half =
       ggI_second_half_times_first_half * gO * second_order_sh +
       ggI_first_half * gO * sig_one_sub_sig;
-  return at::cat({gI_first_half, gI_second_half}, dim);
+  return at::cat({std::move(gI_first_half), std::move(gI_second_half)}, dim);
 }
 
 Tensor glu_double_backward_grad_output(
@@ -2919,7 +2926,8 @@ Tensor as_strided_scatter_backward(
       grad_.new_zeros_symint(input_geometry.sym_sizes())
           .as_strided_symint(
               input_geometry.sym_sizes(), input_geometry.sym_strides());
-  auto result_slice = result.as_strided_symint(sizes, strides, storage_offset);
+  auto result_slice =
+      result.as_strided_symint(sizes, strides, std::move(storage_offset));
   result_slice.copy_(grad_slice);
   return result;
 }
@@ -3014,7 +3022,12 @@ Tensor slice_backward_wrapper(
   auto end_val = end.has_value() ? end.value() : INT64_MAX;
 
   return slice_backward_symint(
-      grad, input_sizes, dim, start_val, end_val, step);
+      grad,
+      input_sizes,
+      dim,
+      std::move(start_val),
+      std::move(end_val),
+      std::move(step));
 }
 
 std::tuple<Tensor, Tensor, Tensor> linalg_svd_jvp(
@@ -3761,7 +3774,9 @@ Tensor differential_analytic_matrix_function(
   // eg. if both are BatchedTensor at different level.
   if (areAnyTensorSubclassLike({A, grad})) {
     meta_grad = at::cat(
-        {at::cat({A, grad}, -1), at::cat({at::zeros_like(A), A}, -1)}, -2);
+        {at::cat({A, grad}, -1),
+         at::cat({at::zeros_like(A), std::move(A)}, -1)},
+        -2);
   } else {
     meta_grad = at::zeros(meta_grad_sizes, grad.options());
     meta_grad.narrow(-2, 0, n).narrow(-1, 0, n).copy_(A);
@@ -4408,7 +4423,7 @@ std::tuple<Tensor, Tensor, Tensor> batchnorm_double_backward(
     ggO = ggO.defined() ? ggO.add_(ggO_G_term) : ggO_G_term;
   }
   if (ggB.defined()) {
-    auto ggO_B_term = ggB_expanded;
+    auto ggO_B_term = std::move(ggB_expanded);
     ggO = ggO.defined() ? ggO.add_(ggO_B_term) : ggO_B_term;
   }
 
@@ -4547,7 +4562,7 @@ std::tuple<Tensor, Tensor, Tensor> layer_norm_double_backward(
     ggO = ggO.defined() ? ggO.add_(ggO_G_term) : ggO_G_term;
   }
   if (ggB.defined()) {
-    auto ggO_B_term = ggB_expanded;
+    auto ggO_B_term = std::move(ggB_expanded);
     ggO = ggO.defined() ? ggO.add_(ggO_B_term) : ggO_B_term;
   }
   if (ggO.defined()) {
@@ -4589,7 +4604,7 @@ infinitely_differentiable_native_group_norm_backward(
   Tensor ds;
   Tensor db;
   if (dY.defined()) {
-    dY_tensor = dY.reshape_symint({N, G, D, HxW});
+    dY_tensor = dY.reshape_symint({N, G, D, std::move(HxW)});
     ds = (dY_tensor * X_tensor).sum(3).unsqueeze_(-1);
     db = dY_tensor.sum(3).unsqueeze_(-1);
   }
@@ -4613,12 +4628,12 @@ infinitely_differentiable_native_group_norm_backward(
       Tensor c = (isDefined(gamma) ? (db * gamma_tensor).sum(2) : db.sum(2))
                      .unsqueeze_(-2);
       b = (c * mean_tensor - b) * rstd_cube * s;
-      c = -b * mean_tensor - c * rstd_tensor * s;
+      c = -b * mean_tensor - c * rstd_tensor * std::move(s);
       dX = a * dY_tensor + b * X_tensor + c;
       if (dmean.defined() && drstd.defined()) {
         dX += var_mean_backward(
             dvar,
-            dmean.view_symint({N, G, 1, 1}),
+            dmean.view_symint({std::move(N), G, 1, 1}),
             X_tensor,
             IntArrayRef{2, 3},
             0,
@@ -4628,7 +4643,7 @@ infinitely_differentiable_native_group_norm_backward(
     } else if (dmean.defined() && drstd.defined()) {
       dX = var_mean_backward(
                dvar,
-               dmean.view_symint({N, G, 1, 1}),
+               dmean.view_symint({std::move(N), G, 1, 1}),
                X_tensor,
                IntArrayRef{2, 3},
                0,
@@ -5463,7 +5478,7 @@ Tensor linalg_lu_solve_jvp(
             /*unitriangular*/ true)
             .matmul(P.mT());
     // dX = op_2(R^H) + S
-    return (left ? R.mH() : R) + S;
+    return (left ? R.mH() : std::move(R)) + S;
   }
 }
 
@@ -5546,7 +5561,7 @@ std::tuple<Tensor, Tensor> linalg_solve_backward(
     gA_ = left ? -gB_.matmul(X_.mH()) : -X_.mH().matmul(gB_);
   }
   return std::make_tuple(
-      A_requires_grad ? gA_ : Tensor{},
+      A_requires_grad ? std::move(gA_) : Tensor{},
       B_requires_grad ? matrix_to_vector(gB_) : Tensor{});
 }
 
@@ -6116,7 +6131,7 @@ Tensor linalg_lu_backward(
         /*left=*/true,
         /*unitriangular=*/true);
 
-    return pivot ? P.matmul(std::move(A_grad)) : A_grad;
+    return pivot ? P.matmul(std::move(A_grad)) : std::move(A_grad);
   } else if (m < n) {
     // Wide case
     // A1_grad = P L^{-H} [U1_grad + (L^H L_grad o 1_L - U_grad U^H o 1_U)
@@ -6275,7 +6290,8 @@ std::tuple<Tensor, Tensor> linalg_lu_jvp(
         at::linalg_solve_triangular(
             L1, PdA2, /*upper=*/false, /*left=*/true, /*unitriangular*/ true) -
         dK.tril(-1).matmul(U2);
-    return std::make_tuple(std::move(dL1), at::cat({dU1, dU2}, /*dim=*/-1));
+    return std::make_tuple(
+        std::move(dL1), at::cat({std::move(dU1), std::move(dU2)}, /*dim=*/-1));
   } else {
     // we only need to update dL2 defined as
     // dL2 := PdA2 U^{-1} - L2 dK.triu()
@@ -6284,7 +6300,8 @@ std::tuple<Tensor, Tensor> linalg_lu_jvp(
     auto dL2 =
         at::linalg_solve_triangular(U1, PdA2, /*upper=*/true, /*left=*/false) -
         L2.matmul(dK.triu());
-    return std::make_tuple(at::cat({dL1, dL2}, /*dim=*/-2), std::move(dU1));
+    return std::make_tuple(
+        at::cat({std::move(dL1), std::move(dL2)}, /*dim=*/-2), std::move(dU1));
   }
 }
 
@@ -6471,7 +6488,7 @@ std::tuple<Tensor, Tensor> scatter_reduce_backward(
       auto node = std::make_shared<DelayedError>(
           "scatter_reduce(): Double backward is unsupported for src when >1 zeros in src are scattered to the same position in self",
           /* num inputs */ 1);
-      auto result = node->apply({grad_src1});
+      auto result = node->apply({std::move(grad_src1)});
       grad_src = result[0];
     } else {
       grad_src = grad_src1;
@@ -6565,7 +6582,7 @@ std::tuple<Tensor, Tensor> index_reduce_backward(
       auto node = std::make_shared<DelayedError>(
           "index_reduce(): Double backward is unsupported for source when >1 zeros in source are scattered to the same position in self",
           /* num inputs */ 1);
-      auto result = node->apply({grad_src1});
+      auto result = node->apply({std::move(grad_src1)});
       grad_src = result[0];
     } else {
       grad_src = grad_src1;

torch/csrc/autograd/VariableTypeManual.cpp

@@ -11,6 +11,8 @@
 #include <torch/csrc/utils/memory.h>
 #include <torch/library.h>
 
+#include <utility>
+
 using namespace at;
 using namespace torch::autograd::generated;
 using torch::autograd::as_view;
@@ -397,7 +399,7 @@ Tensor detach(c10::DispatchKeySet ks, const Tensor& self) {
       /* output */ out,
       /* is_bw_differentiable */ false,
       /* is_fw_differentiable */ false,
-      /* view_func */ func,
+      /* view_func */ std::move(func),
       /* creation_meta */ CreationMeta::DEFAULT,
       /*allow_tensor_metadata_change=*/false);
 
@@ -421,7 +423,7 @@ Tensor _fw_primal(c10::DispatchKeySet ks, const Tensor& self, int64_t level) {
       /* output */ tmp,
       /* is_bw_differentiable */ true,
       /* is_fw_differentiable */ false,
-      /* view_func */ func,
+      /* view_func */ std::move(func),
       /* creation_meta */ CREATION_META_DEFINITION);
 
   return result;
@@ -449,7 +451,7 @@ Tensor _make_dual(
       /* output */ tmp,
       /* is_bw_differentiable */ true,
       /* is_fw_differentiable */ false,
-      /* view_func */ func,
+      /* view_func */ std::move(func),
       /* creation_meta */ CREATION_META_DEFINITION);
 
   return result;