fix(ecs): Implementing backoff and retry for registerScalableTarget a…

…utoscaling request
spinnaker · May 20, 2024 · f576133 · f576133
1 parent f8b8f1e
commit f576133
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 3 deletions.
diff --git a/.../java/com/netflix/spinnaker/clouddriver/ecs/deploy/ops/DisableServiceAtomicOperation.java b/.../java/com/netflix/spinnaker/clouddriver/ecs/deploy/ops/DisableServiceAtomicOperation.java
@@ -21,6 +21,7 @@
 import com.amazonaws.services.ecs.AmazonECS;
 import com.amazonaws.services.ecs.model.UpdateServiceRequest;
 import com.netflix.spinnaker.clouddriver.ecs.deploy.description.ModifyServiceDescription;
+import com.netflix.spinnaker.clouddriver.helpers.OperationPoller;
 import java.util.List;
 
 public class DisableServiceAtomicOperation
@@ -70,7 +71,9 @@ private void disableService() {
                       .withDynamicScalingInSuspended(true)
                       .withDynamicScalingOutSuspended(true)
                       .withScheduledScalingSuspended(true));
-      autoScalingClient.registerScalableTarget(suspendRequest);
+
+      OperationPoller.retryWithBackoff(
+          o -> autoScalingClient.registerScalableTarget(suspendRequest), 1000, 3);
       updateTaskStatus(
           String.format("Autoscaling on server group %s suspended for %s.", service, account));
     }

diff --git a/...n/java/com/netflix/spinnaker/clouddriver/ecs/deploy/ops/EnableServiceAtomicOperation.java b/...n/java/com/netflix/spinnaker/clouddriver/ecs/deploy/ops/EnableServiceAtomicOperation.java
@@ -21,6 +21,7 @@
 import com.amazonaws.services.ecs.AmazonECS;
 import com.amazonaws.services.ecs.model.UpdateServiceRequest;
 import com.netflix.spinnaker.clouddriver.ecs.deploy.description.ModifyServiceDescription;
+import com.netflix.spinnaker.clouddriver.helpers.OperationPoller;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -68,7 +69,9 @@ private void enableService() {
                     .withDynamicScalingInSuspended(false)
                     .withDynamicScalingOutSuspended(false)
                     .withScheduledScalingSuspended(false));
-    autoScalingClient.registerScalableTarget(resumeRequest);
+
+    OperationPoller.retryWithBackoff(
+        o -> autoScalingClient.registerScalableTarget(resumeRequest), 1000, 3);
     updateTaskStatus(
         String.format("Autoscaling on server group %s resumed for %s.", service, account));
   }

diff --git a/...n/java/com/netflix/spinnaker/clouddriver/ecs/deploy/ops/ResizeServiceAtomicOperation.java b/...n/java/com/netflix/spinnaker/clouddriver/ecs/deploy/ops/ResizeServiceAtomicOperation.java
@@ -25,6 +25,7 @@
 import com.amazonaws.services.ecs.model.UpdateServiceRequest;
 import com.netflix.spinnaker.clouddriver.ecs.deploy.description.ResizeServiceDescription;
 import com.netflix.spinnaker.clouddriver.ecs.services.ContainerInformationService;
+import com.netflix.spinnaker.clouddriver.helpers.OperationPoller;
 import com.netflix.spinnaker.clouddriver.orchestration.AtomicOperation;
 import java.util.List;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -89,7 +90,10 @@ private void resizeAutoScalingGroup(Service service) {
         String.format(
             "Resizing Scalable Target of %s to %s instances",
             service.getServiceName(), desiredCount));
-    autoScalingClient.registerScalableTarget(request);
+
+    OperationPoller.retryWithBackoff(
+        o -> autoScalingClient.registerScalableTarget(request), 1000, 3);
+
     updateTaskStatus(
         String.format(
             "Done resizing Scalable Target of %s to %s instances",