Merge branch '4.20' into 4.22

Author: sureshanaparti

api/src/main/java/org/apache/cloudstack/alert/AlertService.java

@@ -71,8 +71,8 @@ private AlertType(short type, String name, boolean isDefault) {
         public static final AlertType ALERT_TYPE_HA_ACTION = new AlertType((short)30, "ALERT.HA.ACTION", true);
         public static final AlertType ALERT_TYPE_CA_CERT = new AlertType((short)31, "ALERT.CA.CERT", true);
         public static final AlertType ALERT_TYPE_VM_SNAPSHOT = new AlertType((short)32, "ALERT.VM.SNAPSHOT", true);
-        public static final AlertType ALERT_TYPE_VR_PUBLIC_IFACE_MTU = new AlertType((short)32, "ALERT.VR.PUBLIC.IFACE.MTU", true);
-        public static final AlertType ALERT_TYPE_VR_PRIVATE_IFACE_MTU = new AlertType((short)32, "ALERT.VR.PRIVATE.IFACE.MTU", true);
+        public static final AlertType ALERT_TYPE_VR_PUBLIC_IFACE_MTU = new AlertType((short)33, "ALERT.VR.PUBLIC.IFACE.MTU", true);
+        public static final AlertType ALERT_TYPE_VR_PRIVATE_IFACE_MTU = new AlertType((short)34, "ALERT.VR.PRIVATE.IFACE.MTU", true);
         public static final AlertType ALERT_TYPE_EXTENSION_PATH_NOT_READY = new AlertType((short)33, "ALERT.TYPE.EXTENSION.PATH.NOT.READY", true);
         public static final AlertType ALERT_TYPE_BACKUP_STORAGE = new AlertType(Capacity.CAPACITY_TYPE_BACKUP_STORAGE, "ALERT.STORAGE.BACKUP", true);
         public static final AlertType ALERT_TYPE_OBJECT_STORAGE = new AlertType(Capacity.CAPACITY_TYPE_OBJECT_STORAGE, "ALERT.STORAGE.OBJECT", true);

api/src/main/java/org/apache/cloudstack/api/command/admin/host/AddSecondaryStorageCmd.java

@@ -29,6 +29,11 @@
 import com.cloud.exception.DiscoveryException;
 import com.cloud.storage.ImageStore;
 import com.cloud.user.Account;
+import org.apache.commons.collections.MapUtils;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
 
 @APICommand(name = "addSecondaryStorage", description = "Adds secondary storage.", responseObject = ImageStoreResponse.class,
         requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
@@ -44,6 +49,9 @@ public class AddSecondaryStorageCmd extends BaseCmd {
     @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, description = "The Zone ID for the secondary storage")
     protected Long zoneId;
 
+    @Parameter(name = ApiConstants.DETAILS, type = CommandType.MAP, description = "Details in key/value pairs using format details[i].keyname=keyvalue. Example: details[0].copytemplatesfromothersecondarystorages=true")
+    protected Map details;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -56,6 +64,20 @@ public Long getZoneId() {
         return zoneId;
     }
 
+    public Map<String, String> getDetails() {
+        Map<String, String> detailsMap = new HashMap<>();
+        if (MapUtils.isNotEmpty(details)) {
+            Collection<?> props = details.values();
+            for (Object prop : props) {
+                HashMap<String, String> detail = (HashMap<String, String>) prop;
+                for (Map.Entry<String, String> entry: detail.entrySet()) {
+                    detailsMap.put(entry.getKey(),entry.getValue());
+                }
+            }
+        }
+        return detailsMap;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////
@@ -68,7 +90,7 @@ public long getEntityOwnerId() {
     @Override
     public void execute(){
         try{
-            ImageStore result = _storageService.discoverImageStore(null, getUrl(), "NFS", getZoneId(), null);
+            ImageStore result = _storageService.discoverImageStore(null, getUrl(), "NFS", getZoneId(), getDetails());
             ImageStoreResponse storeResponse = null;
             if (result != null ) {
                     storeResponse = _responseGenerator.createImageStoreResponse(result);

api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java

@@ -271,8 +271,7 @@ public void execute() {
     }
 
     public Snapshot.LocationType getLocationType() {
-
-        if (Snapshot.LocationType.values() == null || Snapshot.LocationType.values().length == 0 || locationType == null) {
+        if (locationType == null) {
             return null;
         }
 

engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/StorageOrchestrationService.java

@@ -22,7 +22,6 @@
 
 import org.apache.cloudstack.api.response.MigrationResponse;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
-import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult;
 import org.apache.cloudstack.storage.ImageStoreService.MigrationPolicy;
 
@@ -31,5 +30,5 @@ public interface StorageOrchestrationService {
 
     MigrationResponse migrateResources(Long srcImgStoreId, Long destImgStoreId, List<Long> templateIdList, List<Long> snapshotIdList);
 
-    Future<TemplateApiResult> orchestrateTemplateCopyToImageStore(TemplateInfo source, DataStore destStore);
+    Future<TemplateApiResult> orchestrateTemplateCopyFromSecondaryStores(long templateId, DataStore destStore);
 }

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java

@@ -80,4 +80,6 @@ public TemplateInfo getTemplate() {
     List<DatadiskTO> getTemplateDatadisksOnImageStore(TemplateInfo templateInfo, String configurationId);
 
     AsyncCallFuture<TemplateApiResult> copyTemplateToImageStore(DataObject source, DataStore destStore);
-}
+
+    void handleTemplateCopyFromSecondaryStores(long templateId, DataStore destStore);
+ }

engine/components-api/src/main/java/com/cloud/alert/AlertManager.java

@@ -54,5 +54,4 @@ public interface AlertManager extends Manager, AlertService {
     void recalculateCapacity();
 
     void sendAlert(AlertType alertType, long dataCenterId, Long podId, String subject, String body);
-
 }

engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java

@@ -57,8 +57,8 @@ public interface ResourceManager extends ResourceService, Configurable {
 
     ConfigKey<Boolean> KvmSshToAgentEnabled = new ConfigKey<>("Advanced", Boolean.class,
             "kvm.ssh.to.agent","true",
-            "Number of retries when preparing a host into Maintenance Mode is faulty before failing",
-            false);
+            "True if the management server will restart the agent service via SSH into the KVM hosts after or during maintenance operations",
+            true);
 
     ConfigKey<String> HOST_MAINTENANCE_LOCAL_STRATEGY = new ConfigKey<>(String.class,
     "host.maintenance.local.storage.strategy", "Advanced","Error",

engine/components-api/src/main/java/com/cloud/storage/StorageManager.java

@@ -228,8 +228,9 @@ public interface StorageManager extends StorageService {
             ConfigKey.Scope.Global,
             null);
 
-    ConfigKey<Boolean> COPY_PUBLIC_TEMPLATES_FROM_OTHER_STORAGES = new ConfigKey<>(Boolean.class, "copy.public.templates.from.other.storages",
-            "Storage", "true", "Allow SSVMs to try copying public templates from one secondary storage to another instead of downloading them from the source.",
+    ConfigKey<Boolean> COPY_TEMPLATES_FROM_OTHER_SECONDARY_STORAGES = new ConfigKey<>(Boolean.class, "copy.templates.from.other.secondary.storages",
+            "Storage", "true", "When enabled, this feature allows templates to be copied from existing Secondary Storage servers (within the same zone or across zones) " +
+            "while adding a new Secondary Storage. If the copy operation fails, the system falls back to downloading the template from the source URL.",
             true, ConfigKey.Scope.Zone, null);
 
     /**

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java

@@ -36,6 +36,9 @@
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import com.cloud.dc.dao.DataCenterDao;
+import com.cloud.storage.dao.VMTemplateDao;
+import com.cloud.template.TemplateManager;
 import org.apache.cloudstack.api.response.MigrationResponse;
 import org.apache.cloudstack.engine.orchestration.service.StorageOrchestrationService;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
@@ -45,6 +48,7 @@
 import org.apache.cloudstack.engine.subsystem.api.storage.SecondaryStorageService.DataObjectResult;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
+import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult;
@@ -103,6 +107,15 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra
     VolumeDataStoreDao volumeDataStoreDao;
     @Inject
     DataMigrationUtility migrationHelper;
+    @Inject
+    TemplateManager templateManager;
+    @Inject
+    VMTemplateDao templateDao;
+    @Inject
+    TemplateDataFactory templateDataFactory;
+    @Inject
+    DataCenterDao dcDao;
+
 
     ConfigKey<Double> ImageStoreImbalanceThreshold = new ConfigKey<>("Advanced", Double.class,
             "image.store.imbalance.threshold",
@@ -304,8 +317,9 @@ public MigrationResponse migrateResources(Long srcImgStoreId, Long destImgStoreI
     }
 
     @Override
-    public Future<TemplateApiResult> orchestrateTemplateCopyToImageStore(TemplateInfo source, DataStore destStore) {
-        return submit(destStore.getScope().getScopeId(), new CopyTemplateTask(source, destStore));
+    public Future<TemplateApiResult> orchestrateTemplateCopyFromSecondaryStores(long srcTemplateId, DataStore destStore) {
+        Long dstZoneId = destStore.getScope().getScopeId();
+        return submit(dstZoneId, new CopyTemplateFromSecondaryStorageTask(srcTemplateId, destStore));
     }
 
     protected Pair<String, Boolean> migrateCompleted(Long destDatastoreId, DataStore srcDatastore, List<DataObject> files, MigrationPolicy migrationPolicy, int skipped) {
@@ -624,13 +638,13 @@ public DataObjectResult call() {
         }
     }
 
-    private class CopyTemplateTask implements Callable<TemplateApiResult> {
-        private TemplateInfo sourceTmpl;
-        private DataStore destStore;
-        private String logid;
+    private class CopyTemplateFromSecondaryStorageTask implements Callable<TemplateApiResult> {
+        private final long srcTemplateId;
+        private final DataStore destStore;
+        private final String logid;
 
-        public CopyTemplateTask(TemplateInfo sourceTmpl, DataStore destStore) {
-            this.sourceTmpl = sourceTmpl;
+        CopyTemplateFromSecondaryStorageTask(long srcTemplateId, DataStore destStore) {
+            this.srcTemplateId = srcTemplateId;
             this.destStore = destStore;
             this.logid = ThreadContext.get(LOGCONTEXTID);
         }
@@ -639,17 +653,16 @@ public CopyTemplateTask(TemplateInfo sourceTmpl, DataStore destStore) {
         public TemplateApiResult call() {
             ThreadContext.put(LOGCONTEXTID, logid);
             TemplateApiResult result;
-            AsyncCallFuture<TemplateApiResult> future = templateService.copyTemplateToImageStore(sourceTmpl, destStore);
+            long destZoneId = destStore.getScope().getScopeId();
+            TemplateInfo sourceTmpl = templateDataFactory.getTemplate(srcTemplateId, DataStoreRole.Image);
             try {
-                result = future.get();
-            } catch (ExecutionException | InterruptedException e) {
-                logger.warn("Exception while copying template [{}] from image store [{}] to image store [{}]: {}",
-                        sourceTmpl.getUniqueName(), sourceTmpl.getDataStore().getName(), destStore.getName(), e.toString());
+                templateService.handleTemplateCopyFromSecondaryStores(srcTemplateId, destStore);
                 result = new TemplateApiResult(sourceTmpl);
-                result.setResult(e.getMessage());
+            } finally {
+                tryCleaningUpExecutor(destZoneId);
+                ThreadContext.clearAll();
             }
-            tryCleaningUpExecutor(destStore.getScope().getScopeId());
-            ThreadContext.clearAll();
+
             return result;
         }
     }

engine/schema/src/main/java/com/cloud/usage/dao/UsageDao.java

@@ -62,7 +62,7 @@ public interface UsageDao extends GenericDao<UsageVO, Long> {
 
     void saveUsageRecords(List<UsageVO> usageRecords);
 
-    void removeOldUsageRecords(int days);
+    void expungeAllOlderThan(int days, long limitPerQuery);
 
     UsageVO persistUsage(final UsageVO usage);