CKS: create Security Groups for CKS clusters of each account (#8316)

This PR fixes #7684

The security groups contain the same rules for port 22 and 6443, no need to recreate for each CKS cluster.

Author: weizhouapache

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java

@@ -121,9 +121,9 @@
 import com.cloud.network.router.NetworkHelper;
 import com.cloud.network.rules.FirewallRule;
 import com.cloud.network.rules.FirewallRuleVO;
+import com.cloud.network.security.SecurityGroup;
 import com.cloud.network.security.SecurityGroupManager;
 import com.cloud.network.security.SecurityGroupService;
-import com.cloud.network.security.SecurityGroupVO;
 import com.cloud.network.security.SecurityRule;
 import com.cloud.network.vpc.NetworkACL;
 import com.cloud.offering.NetworkOffering;
@@ -1068,38 +1068,25 @@ public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd)
             logAndThrow(Level.ERROR, String.format("Creating Kubernetes cluster failed due to error while finding suitable deployment plan for cluster in zone : %s", zone.getName()));
         }
 
-        SecurityGroupVO securityGroupVO = null;
+        SecurityGroup securityGroup = null;
         if (zone.isSecurityGroupEnabled()) {
-            securityGroupVO = securityGroupManager.createSecurityGroup(KubernetesClusterActionWorker.CKS_CLUSTER_SECURITY_GROUP_NAME.concat(Long.toHexString(System.currentTimeMillis())), "Security group for CKS nodes", owner.getDomainId(), owner.getId(), owner.getAccountName());
-            if (securityGroupVO == null) {
-                throw new CloudRuntimeException(String.format("Failed to create security group: %s", KubernetesClusterActionWorker.CKS_CLUSTER_SECURITY_GROUP_NAME));
-            }
-            List<String> cidrList = new ArrayList<>();
-            cidrList.add(NetUtils.ALL_IP4_CIDRS);
-            securityGroupService.authorizeSecurityGroupRule(securityGroupVO.getId(), NetUtils.TCP_PROTO,
-                    KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_SSH_PORT_SG, KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_SSH_PORT_SG,
-                    null, null, cidrList, null, SecurityRule.SecurityRuleType.IngressRule);
-            securityGroupService.authorizeSecurityGroupRule(securityGroupVO.getId(), NetUtils.TCP_PROTO,
-                    KubernetesClusterActionWorker.CLUSTER_API_PORT, KubernetesClusterActionWorker.CLUSTER_API_PORT,
-                    null, null, cidrList, null, SecurityRule.SecurityRuleType.IngressRule);
-            securityGroupService.authorizeSecurityGroupRule(securityGroupVO.getId(), NetUtils.ALL_PROTO,
-                    null, null, null, null, cidrList, null, SecurityRule.SecurityRuleType.EgressRule);
+            securityGroup = getOrCreateSecurityGroupForAccount(owner);
         }
 
         final Network defaultNetwork = getKubernetesClusterNetworkIfMissing(cmd.getName(), zone, owner, (int)controlNodeCount, (int)clusterSize, cmd.getExternalLoadBalancerIpAddress(), cmd.getNetworkId());
         final VMTemplateVO finalTemplate = getKubernetesServiceTemplate(zone, deployDestination.getCluster().getHypervisorType());
         final long cores = serviceOffering.getCpu() * (controlNodeCount + clusterSize);
         final long memory = serviceOffering.getRamSize() * (controlNodeCount + clusterSize);
 
-        SecurityGroupVO finalSecurityGroupVO = securityGroupVO;
+        final SecurityGroup finalSecurityGroup = securityGroup;
         final KubernetesClusterVO cluster = Transaction.execute(new TransactionCallback<KubernetesClusterVO>() {
             @Override
             public KubernetesClusterVO doInTransaction(TransactionStatus status) {
                 KubernetesClusterVO newCluster = new KubernetesClusterVO(cmd.getName(), cmd.getDisplayName(), zone.getId(), clusterKubernetesVersion.getId(),
                         serviceOffering.getId(), finalTemplate.getId(), defaultNetwork.getId(), owner.getDomainId(),
                         owner.getAccountId(), controlNodeCount, clusterSize, KubernetesCluster.State.Created, cmd.getSSHKeyPairName(), cores, memory, cmd.getNodeRootDiskSize(), "");
                 if (zone.isSecurityGroupEnabled()) {
-                    newCluster.setSecurityGroupId(finalSecurityGroupVO.getId());
+                    newCluster.setSecurityGroupId(finalSecurityGroup.getId());
                 }
                 kubernetesClusterDao.persist(newCluster);
                 return newCluster;
@@ -1114,6 +1101,29 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) {
         return cluster;
     }
 
+    private SecurityGroup getOrCreateSecurityGroupForAccount(Account owner) {
+        String securityGroupName = String.format("%s-%s", KubernetesClusterActionWorker.CKS_CLUSTER_SECURITY_GROUP_NAME, owner.getUuid());
+        String securityGroupDesc = String.format("%s and account %s", KubernetesClusterActionWorker.CKS_SECURITY_GROUP_DESCRIPTION, owner.getName());
+        SecurityGroup securityGroup = securityGroupManager.getSecurityGroup(securityGroupName, owner.getId());
+        if (securityGroup == null) {
+            securityGroup = securityGroupManager.createSecurityGroup(securityGroupName, securityGroupDesc, owner.getDomainId(), owner.getId(), owner.getAccountName());
+            if (securityGroup == null) {
+                throw new CloudRuntimeException(String.format("Failed to create security group: %s", KubernetesClusterActionWorker.CKS_CLUSTER_SECURITY_GROUP_NAME));
+            }
+            List<String> cidrList = new ArrayList<>();
+            cidrList.add(NetUtils.ALL_IP4_CIDRS);
+            securityGroupService.authorizeSecurityGroupRule(securityGroup.getId(), NetUtils.TCP_PROTO,
+                    KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_SSH_PORT_SG, KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_SSH_PORT_SG,
+                    null, null, cidrList, null, SecurityRule.SecurityRuleType.IngressRule);
+            securityGroupService.authorizeSecurityGroupRule(securityGroup.getId(), NetUtils.TCP_PROTO,
+                    KubernetesClusterActionWorker.CLUSTER_API_PORT, KubernetesClusterActionWorker.CLUSTER_API_PORT,
+                    null, null, cidrList, null, SecurityRule.SecurityRuleType.IngressRule);
+            securityGroupService.authorizeSecurityGroupRule(securityGroup.getId(), NetUtils.ALL_PROTO,
+                    null, null, null, null, cidrList, null, SecurityRule.SecurityRuleType.EgressRule);
+        }
+        return securityGroup;
+    }
+
     /**
      * Start operation can be performed at two different life stages of Kubernetes cluster. First when a freshly created cluster
      * in which case there are no resources provisioned for the Kubernetes cluster. So during start all the resources

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java

@@ -106,6 +106,7 @@ public class KubernetesClusterActionWorker {
     public static final int CLUSTER_NODES_DEFAULT_SSH_PORT_SG = DEFAULT_SSH_PORT;
 
     public static final String CKS_CLUSTER_SECURITY_GROUP_NAME = "CKSSecurityGroup";
+    public static final String CKS_SECURITY_GROUP_DESCRIPTION = "Security group for CKS nodes";
 
     protected static final Logger LOGGER = Logger.getLogger(KubernetesClusterActionWorker.class);
 

server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java

@@ -63,7 +63,6 @@
 import com.cloud.agent.api.SecurityGroupRulesCmd.IpPortAndProto;
 import com.cloud.agent.api.to.VirtualMachineTO;
 import com.cloud.agent.manager.Commands;
-import com.cloud.api.query.dao.SecurityGroupJoinDao;
 import com.cloud.configuration.Config;
 import com.cloud.domain.dao.DomainDao;
 import com.cloud.event.ActionEvent;
@@ -131,8 +130,6 @@ public class SecurityGroupManagerImpl extends ManagerBase implements SecurityGro
     @Inject
     SecurityGroupDao _securityGroupDao;
     @Inject
-    SecurityGroupJoinDao _securityGroupJoinDao;
-    @Inject
     SecurityGroupRuleDao _securityGroupRuleDao;
     @Inject
     SecurityGroupVMMapDao _securityGroupVMMapDao;
@@ -1405,7 +1402,7 @@ public boolean isVmSecurityGroupEnabled(Long vmId) {
     }
 
     @Override
-    public SecurityGroupVO getDefaultSecurityGroup(long accountId) {
+    public SecurityGroup getDefaultSecurityGroup(long accountId) {
         return _securityGroupDao.findByAccountAndName(accountId, DEFAULT_GROUP_NAME);
     }