fix systemvm deployment retry

Check plan for a created instance so retry can be done if it fails for a
particular template

Signed-off-by: Abhishek Kumar <[email protected]>

Author: shwstppr

engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java

@@ -18,11 +18,11 @@
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
@@ -595,10 +595,10 @@ private List<VMTemplateVO> listAllReadySystemVMTemplatesWithArch(Long zoneId, Hy
         sc.setParameters("templateType", Storage.TemplateType.SYSTEM);
         sc.setParameters("state", VirtualMachineTemplate.State.Active);
         if (hypervisorType != null && !HypervisorType.Any.equals(hypervisorType)) {
-            sc.setParameters("hypervisorType", List.of(hypervisorType));
+            sc.setParameters("hypervisorType", List.of(hypervisorType).toArray());
         } else {
             sc.setParameters("hypervisorType",
-                    availableHypervisors.stream().map(Pair::first).collect(Collectors.toList()));
+                    availableHypervisors.stream().map(Pair::first).distinct().toArray());
         }
         sc.setJoinParameters("vmTemplateJoinTemplateStoreRef", "downloadState",
                 List.of(VMTemplateStorageResourceAssoc.Status.DOWNLOADED,
@@ -612,7 +612,9 @@ private List<VMTemplateVO> listAllReadySystemVMTemplatesWithArch(Long zoneId, Hy
                 uniqueTemplates.put(key, template);
             }
         }
-        return new ArrayList<>(uniqueTemplates.values());
+        List<VMTemplateVO> result = new ArrayList<>(uniqueTemplates.values());
+        result.sort(Comparator.comparing(VMTemplateVO::getId).reversed());
+        return result;
     }
 
     @Override

server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java

@@ -74,6 +74,7 @@
 import com.cloud.deploy.DataCenterDeployment;
 import com.cloud.deploy.DeployDestination;
 import com.cloud.deploy.DeploymentPlanner;
+import com.cloud.deploy.DeploymentPlanningManager;
 import com.cloud.event.ActionEvent;
 import com.cloud.event.EventTypes;
 import com.cloud.exception.ConcurrentOperationException;
@@ -145,6 +146,7 @@
 import com.cloud.vm.VirtualMachineManager;
 import com.cloud.vm.VirtualMachineName;
 import com.cloud.vm.VirtualMachineProfile;
+import com.cloud.vm.VirtualMachineProfileImpl;
 import com.cloud.vm.dao.ConsoleProxyDao;
 import com.cloud.vm.dao.UserVmDetailsDao;
 import com.cloud.vm.dao.VMInstanceDao;
@@ -227,6 +229,8 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy
     private CAManager caManager;
     @Inject
     private NetworkOrchestrationService networkMgr;
+    @Inject
+    DeploymentPlanningManager deploymentPlanningManager;
 
     private ConsoleProxyListener consoleProxyListener;
 
@@ -719,26 +723,25 @@ protected Map<String, Object> createProxyInstance(long dataCenterId, List<VMTemp
             serviceOffering = serviceOfferingDao.findDefaultSystemOffering(ServiceOffering.consoleProxyDefaultOffUniqueName, ConfigurationManagerImpl.SystemVMUseLocalStorage.valueIn(dataCenterId));
         }
         ConsoleProxyVO proxy = null;
-        InsufficientCapacityException lastException = null;
-        for (VMTemplateVO template : templates) {
+        for (final Iterator<VMTemplateVO> templateIterator = templates.iterator(); templateIterator.hasNext();) {
+            VMTemplateVO template = templateIterator.next();
             proxy = createOrUpdateConsoleProxy(proxy, dataCenterId, id, name, serviceOffering, template, systemAcct);
             try {
                 virtualMachineManager.allocate(name, template, serviceOffering, networks, plan, null);
                 proxy = consoleProxyDao.findById(proxy.getId());
-                lastException = null;
+                final VirtualMachineProfileImpl vmProfile =
+                        new VirtualMachineProfileImpl(proxy, template, serviceOffering, systemAcct, null);
+                deploymentPlanningManager.planDeployment(vmProfile, plan, new DeploymentPlanner.ExcludeList(), null);
                 break;
             } catch (InsufficientCapacityException e) {
-                String message = String.format("Unable to allocate proxy [%s] on zone [%s] with %s due to [%s].",
-                        proxy.toString(), dataCenterId, template, e.getMessage());
-                logger.warn(message, e);
-                lastException = e;
+                if (templateIterator.hasNext()) {
+                    logger.debug("Unable to allocate proxy {} with {} in {} due to [{}]. Retrying with another template", proxy, template, dc, e.getMessage(), e);
+                    continue;
+                }
+                throw new CloudRuntimeException("Failed to allocate proxy [%s] in zone [%s] with available templates", e);
             }
         }
 
-        if (lastException != null) {
-            throw new CloudRuntimeException("Failed to allocate proxy [%s] on zone [%s] with available templates");
-        }
-
         Map<String, Object> context = new HashMap<>();
         context.put("dc", dc);
         HostPodVO pod = hostPodDao.findById(proxy.getPodIdToDeployIn());

services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java

@@ -25,6 +25,7 @@
 import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -82,6 +83,7 @@
 import com.cloud.deploy.DataCenterDeployment;
 import com.cloud.deploy.DeployDestination;
 import com.cloud.deploy.DeploymentPlanner;
+import com.cloud.deploy.DeploymentPlanningManager;
 import com.cloud.event.ActionEvent;
 import com.cloud.event.EventTypes;
 import com.cloud.exception.ConcurrentOperationException;
@@ -155,6 +157,7 @@
 import com.cloud.vm.VirtualMachineManager;
 import com.cloud.vm.VirtualMachineName;
 import com.cloud.vm.VirtualMachineProfile;
+import com.cloud.vm.VirtualMachineProfileImpl;
 import com.cloud.vm.dao.SecondaryStorageVmDao;
 import com.cloud.vm.dao.UserVmDetailsDao;
 import com.cloud.vm.dao.VMInstanceDao;
@@ -254,6 +257,8 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar
     private IndirectAgentLB indirectAgentLB;
     @Inject
     private CAManager caManager;
+    @Inject
+    DeploymentPlanningManager deploymentPlanningManager;
     private int _secStorageVmMtuSize;
 
     private String _instance;
@@ -686,26 +691,26 @@ protected Map<String, Object> createSecStorageVmInstance(long dataCenterId, Seco
             serviceOffering = _offeringDao.findDefaultSystemOffering(ServiceOffering.ssvmDefaultOffUniqueName, ConfigurationManagerImpl.SystemVMUseLocalStorage.valueIn(dataCenterId));
         }
         SecondaryStorageVmVO secStorageVm = null;
-        InsufficientCapacityException lastException = null;
-        for (VMTemplateVO template : templates) {
-            secStorageVm = createOrUpdateSecondaryStorageVm(secStorageVm, id, dataCenterId, name, serviceOffering,
+        for (final Iterator<VMTemplateVO> templateIterator = templates.iterator(); templateIterator.hasNext();) {
+            VMTemplateVO template = templateIterator.next();
+            secStorageVm = createOrUpdateSecondaryStorageVm(secStorageVm, dataCenterId, id, name, serviceOffering,
                     template, systemAcct, role);
             try {
                 _itMgr.allocate(name, template, serviceOffering, networks, plan, null);
                 secStorageVm = _secStorageVmDao.findById(secStorageVm.getId());
-                lastException = null;
+                final VirtualMachineProfileImpl vmProfile =
+                        new VirtualMachineProfileImpl(secStorageVm, template, serviceOffering, systemAcct, null);
+                deploymentPlanningManager.planDeployment(vmProfile, plan, new DeploymentPlanner.ExcludeList(), null);
                 break;
             } catch (InsufficientCapacityException e) {
-                String errorMessage = String.format("Unable to allocate secondary storage VM [%s] due to [%s].", name, e.getMessage());
-                logger.warn(errorMessage, e);
-                lastException = e;
+                if (templateIterator.hasNext()) {
+                    logger.debug("Unable to allocate secondary storage {} with {} due to [{}]. Retrying with another template", secStorageVm, template, e.getMessage(), e);
+                    continue;
+                }
+                throw new CloudRuntimeException("Failed to allocate secondary storage VM [%s] in zone [%s] with available templates", e);
             }
         }
 
-        if (lastException != null) {
-            throw new CloudRuntimeException("Failed to allocate secondary storage VM [%s] on zone [%s] with available templates");
-        }
-
         Map<String, Object> context = new HashMap<>();
         context.put("secStorageVmId", secStorageVm.getId());
         return context;