Add validate url check while reading thread intel feed (#1660)

.github/workflows/ci.yml

@@ -49,7 +49,7 @@ jobs:
       - name: Build and Test
         run: |
           chown -R 1000:1000 `pwd`
-          su `id -un 1000` -c "whoami && java -version && ./gradlew build"
+          su `id -un 1000` -c "whoami && java -version && ./gradlew build -Dbuild.snapshot=false -Dopensearch.version=3.1.0"
 
       - name: Create Artifact Path
         run: |
@@ -112,7 +112,7 @@ jobs:
 
       - name: Build and Test
         working-directory: ${{ env.WORKING_DIR }}
-        run: ./gradlew build ${{ env.BUILD_ARGS }}
+        run: ./gradlew -Dbuild.snapshot=false -Dopensearch.version=3.1.0 build ${{ env.BUILD_ARGS }}
         env:
           _JAVA_OPTIONS: ${{ matrix.os_java_options }}
 

.github/workflows/security-test-workflow.yml

@@ -49,7 +49,7 @@ jobs:
       - name: Run integration tests
         run: |
           chown -R 1000:1000 `pwd`
-          su `id -un 1000` -c "./gradlew integTest -Dsecurity=true -Dhttps=true --tests '*IT'"
+          su `id -un 1000` -c "./gradlew integTest -Dbuild.snapshot=false -Dsecurity=true -Dopensearch.version=3.1.0 -Dhttps=true --tests '*IT'"
 
       - name: Upload failed logs
         uses: actions/upload-artifact@v4

build.gradle

@@ -33,8 +33,8 @@ buildscript {
         }
 
         alerting_spi_build = opensearch_build
-        alerting_spi_build += "-SNAPSHOT"
         if (isSnapshot) {
+            alerting_spi_build += "-SNAPSHOT"
             opensearch_build += "-SNAPSHOT"
 
             // TODO consider enabling snapshot options once SA commons is published to maven central

src/main/java/org/opensearch/securityanalytics/threatIntel/model/UrlDownloadSource.java

@@ -1,5 +1,7 @@
 package org.opensearch.securityanalytics.threatIntel.model;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -9,11 +11,13 @@
 
 import java.io.IOException;
 import java.net.URL;
+import java.util.Locale;
 
 /**
  * This is a Threat Intel Source config where the iocs are downloaded from the URL
  */
 public class UrlDownloadSource extends Source implements Writeable, ToXContent {
+    private static final Logger log = LogManager.getLogger(UrlDownloadSource.class);
     public static final String URL_FIELD = "url";
     public static final String FEED_FORMAT_FIELD = "feed_format";
     public static final String HAS_CSV_HEADER_FIELD = "has_csv_header_field";
@@ -71,6 +75,11 @@ public static UrlDownloadSource parse(XContentParser xcp) throws IOException {
                 case URL_FIELD:
                     String urlString = xcp.text();
                     url = new URL(urlString);
+                    String protocol = url.getProtocol().toLowerCase(Locale.ROOT);
+                    if (!"http".equals(protocol) && !"https".equals(protocol)) {
+                        log.error("Unsupported protocol [{}]. Only http and https are allowed. Url:{}", protocol, urlString);
+                        throw new IOException("Unsupported protocol [" + protocol + "]. Only http and https are allowed.");
+                    }
                     break;
                 case FEED_FORMAT_FIELD:
                     feedFormat = xcp.text();

src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java

@@ -17,10 +17,13 @@
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.net.InetAddress;
 import java.net.URL;
 import java.net.URLConnection;
+import java.net.UnknownHostException;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
+import java.util.Locale;
 
 //Parser helper class
 public class ThreatIntelFeedParser {
@@ -34,6 +37,11 @@ public class ThreatIntelFeedParser {
      */
     @SuppressForbidden(reason = "Need to connect to http endpoint to read threat intel feed database file")
     public static CSVParser getThreatIntelFeedReaderCSV(final TIFMetadata tifMetadata) {
+        try {
+            validateUrl(new URL(tifMetadata.getUrl()));
+        } catch (IOException e) {
+            throw new OpenSearchException("Invalid threat intel feed URL [{}]", tifMetadata.getUrl(), e);
+        }
         SpecialPermission.check();
         return AccessController.doPrivileged((PrivilegedAction<CSVParser>) () -> {
             try {
@@ -53,6 +61,7 @@ public static CSVParser getThreatIntelFeedReaderCSV(final TIFMetadata tifMetadat
      */
     @SuppressForbidden(reason = "Need to connect to http endpoint to read threat intel feed database file")
     public static CSVParser getThreatIntelFeedReaderCSV(URL url) {
+        validateUrl(url);
         SpecialPermission.check();
         return AccessController.doPrivileged((PrivilegedAction<CSVParser>) () -> {
             try {
@@ -65,4 +74,32 @@ public static CSVParser getThreatIntelFeedReaderCSV(URL url) {
             }
         });
     }
+
+    private static void validateUrl(URL url) {
+        String protocol = url.getProtocol().toLowerCase(Locale.ROOT);
+        if (!"http".equals(protocol) && !"https".equals(protocol)) {
+            log.error("Unsupported protocol [{}]. Only http and https are allowed.", protocol);
+            throw new OpenSearchException("Unsupported protocol [{}]. Only http and https are allowed.", protocol);
+        }
+
+        InetAddress address;
+        try {
+            address = InetAddress.getByName(url.getHost());
+        } catch (UnknownHostException e) {
+            log.error("Unable to resolve host [{}]", url.getHost());
+            throw new OpenSearchException("Unable to resolve host [{}]", url.getHost());
+        }
+
+        if (address.isLoopbackAddress()
+                || address.isLinkLocalAddress()
+                || address.isSiteLocalAddress()
+                || address.isAnyLocalAddress()) {
+            log.error("URL [{}] points to a restricted address. Loopback, link-local, and private addresses are not allowed.",
+                    url);
+            throw new OpenSearchException(
+                    "URL [{}] points to a restricted address. Loopback, link-local, and private addresses are not allowed.",
+                    url
+            );
+        }
+    }
 }

src/test/java/org/opensearch/securityanalytics/threatIntel/model/ThreatIntelSourceTests.java

@@ -5,6 +5,7 @@
 
 package org.opensearch.securityanalytics.threatIntel.model;
 
+import org.apache.commons.lang3.tuple.Pair;
 import org.junit.Test;
 import org.opensearch.core.rest.RestStatus;
 import org.opensearch.securityanalytics.TestHelpers;
@@ -70,4 +71,26 @@ public void testParseInvalidSourceField() {
         assertEquals(RestStatus.BAD_REQUEST, exception.status());
         assertTrue(exception.getMessage().contains("Unexpected input in 'source' field when reading ioc store config."));
     }
+
+    @Test
+    public void testParseWithUrlDownloadSource_fileProtocolBlocked() {
+        Pair<String, String>[] blockedUrls = new Pair[] {
+                Pair.of("file:///etc/passwd", "file"),
+                Pair.of("ftp://example.com/feed.csv", "ftp"),
+                Pair.of("jar:file:///tmp/test.jar!/", "jar")
+        };
+
+        for (Pair<String, String> blockedUrl : blockedUrls) {
+            String sourceString = "{\n" +
+                    "  \"url_download\": {\n" +
+                    "    \"url\": \"" + blockedUrl.getLeft() + "\",\n" +
+                    "    \"feed_format\": \"csv\"\n" +
+                    "  }\n" +
+                    "}";
+            Exception e = assertThrows(IOException.class,
+                    () -> Source.parse(TestHelpers.parser(sourceString)));
+            assertEquals(String.format("Unsupported protocol [%s]. Only http and https are allowed.", blockedUrl.getRight()),
+                    e.getMessage());
+        }
+    }
 }

src/test/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParserTests.java

@@ -0,0 +1,52 @@
+package org.opensearch.securityanalytics.threatIntel.util;
+
+import org.junit.Test;
+import org.opensearch.OpenSearchException;
+import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.net.URL;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class ThreatIntelFeedParserTests extends OpenSearchTestCase {
+
+    @Test
+    public void testGetThreatIntelFeedReaderCSV_blockedUrls() {
+        String[] blockedUrls = {
+                "file:///etc/passwd", //fileProtocolBlocked
+                "ftp://example.com/feed.csv", //ftpProtocolBlocked
+                "http://127.0.0.1:9200", //loopbackBlocked
+                "http://localhost:9200", //localhostBlocked
+                "http://169.254.169.254/latest/meta-data/", //linkLocalBlocked
+                "http://10.0.0.1/feed.csv", //siteLocalBlocked
+                "http://192.168.1.1/feed.csv", //privateNetworkBlocked
+                "jar:file:///tmp/test.jar!/" //jarProtocolBlocked
+        };
+
+        for (String blockedUrl : blockedUrls) {
+            expectThrows(OpenSearchException.class,
+                    () -> ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(new URL(blockedUrl)));
+        }
+    }
+
+    @Test
+    public void testGetThreatIntelFeedReaderCSV_tifMetadata_blockedUrls() {
+        String[] blockedUrls = {
+                "file:///etc/passwd",
+                "http://127.0.0.1:9200",
+                "http://localhost:9200",
+                "http://169.254.169.254/latest/meta-data/",
+                "http://10.0.0.1/feed.csv",
+                "http://192.168.1.1/feed.csv"
+        };
+
+        for (String blockedUrl : blockedUrls) {
+            TIFMetadata tifMetadata = mock(TIFMetadata.class);
+            when(tifMetadata.getUrl()).thenReturn(blockedUrl);
+            expectThrows(OpenSearchException.class,
+                    () -> ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(tifMetadata));
+        }
+    }
+}