Fix direct download URL checks (#7693)

This PR fixes the URL check for direct downloads, in the case of HTTPS URLs the certificates were not loaded into the SSL context

Author: nvazquez

agent/src/main/java/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java

@@ -0,0 +1,83 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package org.apache.cloudstack.direct.download;
+
+import com.cloud.utils.exception.CloudRuntimeException;
+import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.HttpDirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.HttpsDirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.MetalinkDirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.NfsDirectDownloadCommand;
+import org.apache.log4j.Logger;
+
+public class DirectDownloadHelper {
+
+    public static final Logger LOGGER = Logger.getLogger(DirectDownloadHelper.class.getName());
+
+    /**
+     * Get direct template downloader from direct download command and destination pool
+     */
+    public static DirectTemplateDownloader getDirectTemplateDownloaderFromCommand(DirectDownloadCommand cmd,
+                                                                                  String destPoolLocalPath,
+                                                                                  String temporaryDownloadPath) {
+        if (cmd instanceof HttpDirectDownloadCommand) {
+            return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, cmd.getChecksum(), cmd.getHeaders(),
+                    cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath);
+        } else if (cmd instanceof HttpsDirectDownloadCommand) {
+            return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, cmd.getChecksum(), cmd.getHeaders(),
+                    cmd.getConnectTimeout(), cmd.getSoTimeout(), cmd.getConnectionRequestTimeout(), temporaryDownloadPath);
+        } else if (cmd instanceof NfsDirectDownloadCommand) {
+            return new NfsDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), cmd.getChecksum(), temporaryDownloadPath);
+        } else if (cmd instanceof MetalinkDirectDownloadCommand) {
+            return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders(),
+                    cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath);
+        } else {
+            throw new IllegalArgumentException("Unsupported protocol, please provide HTTP(S), NFS or a metalink");
+        }
+    }
+
+    public static boolean checkUrlExistence(String url) {
+        try {
+            DirectTemplateDownloader checker = getCheckerDownloader(url);
+            return checker.checkUrl(url);
+        } catch (CloudRuntimeException e) {
+            LOGGER.error(String.format("Cannot check URL %s is reachable due to: %s", url, e.getMessage()), e);
+            return false;
+        }
+    }
+
+    private static DirectTemplateDownloader getCheckerDownloader(String url) {
+        if (url.toLowerCase().startsWith("https:")) {
+            return new HttpsDirectTemplateDownloader(url);
+        } else if (url.toLowerCase().startsWith("http:")) {
+            return new HttpDirectTemplateDownloader(url);
+        } else if (url.toLowerCase().startsWith("nfs:")) {
+            return new NfsDirectTemplateDownloader(url);
+        } else if (url.toLowerCase().endsWith(".metalink")) {
+            return new MetalinkDirectTemplateDownloader(url);
+        } else {
+            throw new CloudRuntimeException(String.format("Cannot find a download checker for url: %s", url));
+        }
+    }
+
+    public static Long getFileSize(String url, String format) {
+        DirectTemplateDownloader checker = getCheckerDownloader(url);
+        return checker.getRemoteFileSize(url, format);
+    }
+}

agent/src/main/java/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java

@@ -0,0 +1,61 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package org.apache.cloudstack.direct.download;
+
+import com.cloud.utils.Pair;
+
+import java.util.List;
+
+public interface DirectTemplateDownloader {
+
+    /**
+     * Perform template download to pool specified on downloader creation
+     * @return (true if successful, false if not, download file path)
+     */
+    Pair<Boolean, String> downloadTemplate();
+
+    /**
+     * Perform checksum validation of previously downloaded template
+     * @return true if successful, false if not
+     */
+    boolean validateChecksum();
+
+    /**
+     * Validate if the URL is reachable and returns HTTP.OK status code
+     * @return true if the URL is reachable, false if not
+     */
+    boolean checkUrl(String url);
+
+    /**
+     * Obtain the remote file size (and virtual size in case format is qcow2)
+     */
+    Long getRemoteFileSize(String url, String format);
+
+    /**
+     * Get list of urls within metalink content ordered by ascending priority
+     * (for those which priority tag is not defined, highest priority value is assumed)
+     */
+    List<String> getMetalinkUrls(String metalinkUrl);
+
+    /**
+     * Get the list of checksums within a metalink content
+     */
+    List<String> getMetalinkChecksums(String metalinkUrl);
+}