<fix>[kvm]:Check KVM agent port availability

This commit adds a check for the availability of KVM agent port after configuring iptables.
The iptables rules are updated to allow incoming traffic on the KVM agent port,
and then a simple port check is performed to ensure that the port is open and accessible.
This check is important to ensure that the KVM virtualization environment is
properly configured and ready for use.

If the kvmagent port is blocked and the echo fails, the kvmagent
will be restarted and the command will be reissued.
However, due to the kvmagent port being blocked, the command will
fail to be issued.
The HA fencer is unable to update the heartbeat, and during the HA checker
 check, the heartbeat will not be refreshed, potentially resulting in
split-brain VMs.

Resolves: ZSTAC-56336

Change-Id: I6c6578706c6d716f7774776b65776a6f666a6263

Author: ZqyDaoDao

conf/globalConfig/kvm.xml

@@ -154,6 +154,14 @@
         <defaultValue>2</defaultValue>
     </config>
 
+    <config>
+        <category>kvm</category>
+        <name>testKvmagentPortOnConnectTimeout</name>
+        <description>use with testKvmagentPortOnConnectTimeout, the connect timeout launching a socket connection to hosts' kvmagent port, in seconds</description>
+        <type>java.lang.Integer</type>
+        <defaultValue>2</defaultValue>
+    </config>
+
     <config>
         <category>kvm</category>
         <name>restartagentwhenfakedead</name>

plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java

@@ -66,6 +66,8 @@ public class KVMGlobalConfig {
     public static GlobalConfig TEST_SSH_PORT_ON_OPEN_TIMEOUT = new GlobalConfig(CATEGORY, "testSshPortOpenTimeout");
     @GlobalConfigValidation(numberGreaterThan = 0, numberLessThan = 300)
     public static GlobalConfig TEST_SSH_PORT_ON_CONNECT_TIMEOUT = new GlobalConfig(CATEGORY, "testSshPortOnConnectTimeout");
+    @GlobalConfigValidation(numberGreaterThan = 0, numberLessThan = 300)
+    public static GlobalConfig TEST_KVMAGENT_PORT_ON_CONNECT_TIMEOUT = new GlobalConfig(CATEGORY, "testKvmagentPortOnConnectTimeout");
     @GlobalConfigValidation
     public static GlobalConfig RESTART_AGENT_IF_FAKE_DEAD = new GlobalConfig(CATEGORY, "restartagentwhenfakedead");
 

plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java

@@ -91,7 +91,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
+import java.net.*;
 import java.nio.charset.StandardCharsets;
 import java.util.*;
 import java.util.concurrent.TimeUnit;
@@ -4240,6 +4240,32 @@ private void recreateTag(SystemTag tag, String token, String value, boolean inhe
         creator.create();
     }
 
+    public ErrorCode isRemotePortOpen(String ip, int port, int timeout) {
+        Socket socket = new Socket();
+
+        try {
+            socket.setReuseAddress(true);
+            SocketAddress sa = new InetSocketAddress(ip, port);
+            socket.connect(sa, timeout);
+            if (socket.isConnected()) {
+                return null;
+            }
+            return operr("connect remote port[ip:%s, port:%s] is failed", ip, port);
+        } catch (SocketException e) {
+            logger.debug(String.format("unable to connect remote port[ip:%s, port:%s], %s", ip, port, e.getMessage()));
+            return operr("unable to connect remote port[ip:%s, port:%s], %s", ip, port, e.getMessage());
+        } catch (IOException e) {
+            logger.debug(String.format("unable to connect remote port[ip:%s, port:%s], %s", ip, port, e.getMessage()));
+            return operr("unable to connect remote port[ip:%s, port:%s], %s", ip, port, e.getMessage());
+        } finally {
+            try {
+                socket.close();
+            } catch (IOException e) {
+                logger.warn(e.getMessage(), e);
+            }
+        }
+    }
+
     @Override
     public void connectHook(final ConnectHostInfo info, final Completion complete) {
         if (!info.isNewAdded()) {
@@ -4677,6 +4703,26 @@ public void run(FlowTrigger trigger, Map data) {
                     }
                 });
 
+                flow(new NoRollbackFlow() {
+                    String __name__ = "test-kvmagent-port-open";
+
+                    @Override
+                    public boolean skip(Map data) {
+                        return CoreGlobalProperty.UNIT_TEST_ON;
+                    }
+
+                    @Override
+                    public void run(FlowTrigger trigger, Map data) {
+                        long ctimeout = TimeUnit.SECONDS.toMillis(KVMGlobalConfig.TEST_KVMAGENT_PORT_ON_CONNECT_TIMEOUT.value(Integer.class).longValue());
+                        ErrorCode errorCode = isRemotePortOpen(getSelf().getManagementIp(), KVMGlobalProperty.AGENT_PORT, (int) ctimeout);
+                        if (errorCode != null) {
+                            trigger.fail(errorCode);
+                            return;
+                        }
+                        trigger.next();
+                    }
+                });
+
                 flow(new NoRollbackFlow() {
                     String __name__ = "echo-host";
 

test/src/test/resources/globalConfig/kvm.xml

@@ -154,6 +154,14 @@
         <defaultValue>2</defaultValue>
     </config>
 
+    <config>
+        <category>kvm</category>
+        <name>testKvmagentPortOnConnectTimeout</name>
+        <description>use with testKvmagentPortOnConnectTimeout, the connect timeout launching a socket connection to hosts' kvmagent port, in seconds</description>
+        <type>java.lang.Integer</type>
+        <defaultValue>2</defaultValue>
+    </config>
+
     <config>
         <category>kvm</category>
         <name>restartagentwhenfakedead</name>