<fix>[compute]: Fix cancelLongJob error

The retry mechanism is added to the cancellation of the hot migration
host. The default retry is 3 times, and the interval between each time
is 1s.

Resolves: ZSTAC-53214

Change-Id: I7174900153214677a6b74646a786c7065696b62

<fix>[compute]: Fix cancelLongJob error

Cancel the migration host task, if the result of the migration task is
canceled, the cancellation task is successful

Update internal message parameter name

Resolves: ZSTAC-53214

Change-Id: I7174900153214677a6b74646a786c7062696b62

Author: liang-hanyu

compute/src/main/java/org/zstack/compute/host/HostManagerImpl.java

@@ -510,6 +510,8 @@ private void handle(CancelHostTasksMsg msg) {
             nmsg.setHostUuids(msg.getHostUuids());
             nmsg.setSearchedMnIds(msg.getSearchedMnIds());
             nmsg.setCancellationApiId(msg.getCancellationApiId());
+            nmsg.setInterval(msg.getInterval());
+            nmsg.setTimes(msg.getTimes());
             bus.makeServiceIdByManagementNodeId(nmsg, HostConstant.SERVICE_ID, restMnIds.get(0));
             bus.send(nmsg, new CloudBusCallBack(msg) {
                 @Override
@@ -534,6 +536,8 @@ public void run(MessageReply r) {
             CancelHostTaskMsg cmsg = new CancelHostTaskMsg();
             cmsg.setHostUuid(hostUuid);
             cmsg.setCancellationApiId(msg.getCancellationApiId());
+            cmsg.setInterval(msg.getInterval());
+            cmsg.setTimes(msg.getTimes());
             bus.makeLocalServiceId(cmsg, HostConstant.SERVICE_ID);
             bus.send(cmsg, new CloudBusCallBack(compl) {
                 @Override

compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java

@@ -6672,6 +6672,8 @@ protected void handle(CancelMigrateVmMsg msg) {
 
         CancelHostTasksMsg cmsg = new CancelHostTasksMsg();
         cmsg.setCancellationApiId(msg.getCancellationApiId());
+        cmsg.setInterval(1);
+        cmsg.setTimes(3);
         bus.makeLocalServiceId(cmsg, HostConstant.SERVICE_ID);
         bus.send(cmsg, new CloudBusCallBack(msg) {
             @Override

header/src/main/java/org/zstack/header/host/CancelHostTaskMsg.java

@@ -7,7 +7,26 @@
  */
 public class CancelHostTaskMsg extends CancelMessage implements HostMessage {
     private String hostUuid;
+    private Integer times;
+    private Integer interval;
 
+    public Integer getInterval() {
+        return interval;
+    }
+
+    public void setInterval(Integer interval) {
+        this.interval = interval;
+    }
+
+    public Integer getTimes() {
+        return times;
+    }
+
+    public void setTimes(Integer times) {
+        this.times = times;
+    }
+
+    @Override
     public String getHostUuid() {
         return hostUuid;
     }

header/src/main/java/org/zstack/header/host/CancelHostTasksMsg.java

@@ -13,6 +13,24 @@
 public class CancelHostTasksMsg extends CancelMessage {
     private List<String> searchedMnIds = new ArrayList<>();
     private List<String> hostUuids = new ArrayList<>();
+    private Integer times;
+    private Integer interval;
+
+    public Integer getInterval() {
+        return interval;
+    }
+
+    public void setInterval(Integer interval) {
+        this.interval = interval;
+    }
+
+    public Integer getTimes() {
+        return times;
+    }
+
+    public void setTimes(Integer times) {
+        this.times = times;
+    }
 
     public List<String> getSearchedMnIds() {
         return searchedMnIds;

header/src/main/java/org/zstack/header/longjob/LongJobConstants.java

@@ -7,6 +7,8 @@ public class LongJobConstants {
     public static final String SERVICE_ID = "longjob";
     public static final String ACTION_CATEGORY = "longjob";
 
+    public static final String NO_JOB_TO_CANCEL = "no matched job to cancel";
+
     public enum LongJobOperation {
         Start,
         Resume,

longjob/src/main/java/org/zstack/longjob/LongJobManagerImpl.java

@@ -335,6 +335,11 @@ public void success(Boolean cancelled) {
 
             @Override
             public void fail(ErrorCode errorCode) {
+                if (Q.New(LongJobVO.class).eq(LongJobVO_.uuid, vo.getUuid()).eq(LongJobVO_.state, LongJobState.Canceled).isExists()){
+                    completion.success();
+                    return;
+                }
+
                 logger.error(String.format("failed to cancel longjob [uuid:%s, name:%s]", vo.getUuid(), vo.getName()));
                 completion.fail(errorCode);
             }

plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java

@@ -3575,6 +3575,24 @@ public static class PrimaryStorageCommand extends AgentCommand {
 
     public static class CancelCmd extends AgentCommand implements CancelCommand {
         private String cancellationApiId;
+        private Integer times;
+        private Integer interval;
+
+        public Integer getTimes() {
+            return times;
+        }
+
+        public void setTimes(Integer times) {
+            this.times = times;
+        }
+
+        public Integer getInterval() {
+            return interval;
+        }
+
+        public void setInterval(Integer interval) {
+            this.interval = interval;
+        }
 
         @Override
         public void setCancellationApiId(String cancellationApiId) {

plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java

@@ -4554,7 +4554,7 @@ public String getName() {
 
     private void handle(final CancelHostTaskMsg msg) {
         CancelHostTaskReply reply = new CancelHostTaskReply();
-        cancelJob(msg.getCancellationApiId(), new Completion(msg) {
+        cancelJob(msg, new Completion(msg) {
             @Override
             public void success() {
                 bus.reply(msg, reply);
@@ -4568,9 +4568,13 @@ public void fail(ErrorCode errorCode) {
         });
     }
 
-    private void cancelJob(String apiId, Completion completion) {
+    private void cancelJob(CancelHostTaskMsg msg, Completion completion) {
         CancelCmd cmd = new CancelCmd();
-        cmd.setCancellationApiId(apiId);
+        cmd.setCancellationApiId(msg.getCancellationApiId());
+        if (msg.getInterval() != null && msg.getTimes() != null) {
+            cmd.setInterval(msg.getInterval());
+            cmd.setTimes(msg.getTimes());
+        }
         new Http<>(cancelJob, cmd, CancelRsp.class).call(new ReturnValueCompletion<CancelRsp>(completion) {
             @Override
             public void success(CancelRsp ret) {

test/src/test/groovy/org/zstack/test/integration/longjob/LiveMigrateVmJobCase.groovy

@@ -13,6 +13,7 @@ import org.zstack.header.allocator.HostCapacityVO
 import org.zstack.header.allocator.HostCapacityVO_
 import org.zstack.header.host.HostVO
 import org.zstack.header.host.HostVO_
+import org.zstack.header.longjob.LongJobConstants
 import org.zstack.header.longjob.LongJobVO
 import org.zstack.header.longjob.LongJobVO_
 import org.zstack.header.network.service.NetworkServiceType
@@ -28,7 +29,6 @@ import org.zstack.test.integration.ZStackTest
 import org.zstack.testlib.EnvSpec
 import org.zstack.testlib.SubCase
 import org.zstack.utils.data.SizeUnit
-import org.zstack.utils.gson.JSONObjectUtil
 
 /**
  * Created by camile on 18-3-7.
@@ -190,6 +190,7 @@ class LiveMigrateVmJobCase extends SubCase {
             testLiveMigrateVmLongJobCancel()
             testLiveMigrateVmLongJobCancelFail()
             testLiveMigrateVmLongJobCancelBeforeMigrate()
+            testLiveMigrateVmNoJobToCancel()
         }
     }
 
@@ -294,6 +295,68 @@ class LiveMigrateVmJobCase extends SubCase {
         assert canceled
     }
 
+    void testLiveMigrateVmNoJobToCancel() {
+        env.cleanSimulatorHandlers()
+
+        APIMigrateVmMsg msg = new APIMigrateVmMsg()
+        msg.hostUuid = host2.uuid
+        msg.vmInstanceUuid = vm1.uuid
+        def canceled = false
+        def migrating = false
+
+        env.simulator(KVMConstant.KVM_MIGRATE_VM_PATH) { HttpEntity<String> e ->
+            migrating = true
+            while (!canceled) {
+                sleep(500)
+            }
+            return new KVMAgentCommands.MigrateVmResponse()
+        }
+        KVMAgentCommands.CancelCmd cmd = null
+        env.simulator(AgentConstant.CANCEL_JOB) { HttpEntity<String> e ->
+            cmd = json(e.body, KVMAgentCommands.CancelCmd.class)
+            canceled = true
+            def rsp = new KVMAgentCommands.CancelRsp()
+            rsp.setError(LongJobConstants.NO_JOB_TO_CANCEL)
+            return rsp
+        }
+
+        LongJobInventory jobInv = submitLongJob {
+            jobName = msg.getClass().getSimpleName()
+            jobData = gson.toJson(msg)
+        } as LongJobInventory
+
+        assert jobInv.jobName == msg.getClass().getSimpleName()
+        assert jobInv.state == LongJobState.Running
+
+        while (!migrating) {
+            sleep(500)
+        }
+        expectError {
+            cancelLongJob {
+                uuid = jobInv.uuid
+            }
+        }
+
+        retryInSecs() {
+            LongJobVO job = dbFindByUuid(jobInv.getUuid(), LongJobVO.class)
+            assert job.state.toString() == LongJobState.Succeeded.toString()
+        }
+        assert host2.uuid == dbf.findByUuid(vm1.uuid, VmInstanceVO.class).hostUuid
+        assert canceled
+        assert cmd.getInterval() == 1
+        assert cmd.getTimes() == 3
+
+        SQL.New(LongJobVO.class).eq(LongJobVO_.uuid, jobInv.getUuid()).set(LongJobVO_.state, org.zstack.header.longjob.LongJobState.Canceled).update();
+
+        cancelLongJob {
+            uuid = jobInv.uuid
+        }
+
+        SQL.New(LongJobVO.class).eq(LongJobVO_.uuid, jobInv.getUuid()).set(LongJobVO_.state, org.zstack.header.longjob.LongJobState.Succeeded).update();
+
+        env.cleanSimulatorHandlers()
+    }
+
     void testLiveMigrateVmLongJobCancelFail() {
         APIMigrateVmMsg msg = new APIMigrateVmMsg()
         msg.hostUuid = host1.uuid
@@ -366,7 +429,7 @@ class LiveMigrateVmJobCase extends SubCase {
             return new KVMAgentCommands.MigrateVmResponse()
         }
 
-        env.message(AllocateHostMsg.class){ AllocateHostMsg amsg, CloudBus bus ->
+        env.message(AllocateHostMsg.class) { AllocateHostMsg amsg, CloudBus bus ->
             while (!canceled) {
                 sleep(500)
             }
@@ -376,7 +439,7 @@ class LiveMigrateVmJobCase extends SubCase {
             long afterAllocateMemInByte = mem - instance.memorySize
             long afterAllocateCpuCount = cpu - instance.cpuNum
 
-            SQL.New(HostCapacityVO.class).eq(HostCapacityVO_.uuid,  host2.uuid)
+            SQL.New(HostCapacityVO.class).eq(HostCapacityVO_.uuid, host2.uuid)
                     .set(HostCapacityVO_.availableMemory, afterAllocateMemInByte)
                     .set(HostCapacityVO_.availableCpu, afterAllocateCpuCount)
                     .update()