From 28d4cb0cb2456954acf18a94e0ba688a2b6d98ee Mon Sep 17 00:00:00 2001 From: nishtham-amazon Date: Thu, 26 Mar 2026 17:21:05 -0700 Subject: [PATCH] Add validate url check while reading thread intel feed (#1660) * Add validate url check while reading thread intel feed Signed-off-by: nishtham * Added unit tests for url validation for threat intel feeds --------- Signed-off-by: nishtham --- .../threatIntel/model/UrlDownloadSource.java | 9 ++++ .../util/ThreatIntelFeedParser.java | 37 +++++++++++++ .../model/ThreatIntelSourceTests.java | 23 ++++++++ .../util/ThreatIntelFeedParserTests.java | 52 +++++++++++++++++++ 4 files changed, 121 insertions(+) create mode 100644 src/test/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParserTests.java diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/UrlDownloadSource.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/UrlDownloadSource.java index fdc2d9756..d1fa32bb7 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/UrlDownloadSource.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/UrlDownloadSource.java @@ -1,5 +1,7 @@ package org.opensearch.securityanalytics.threatIntel.model; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -9,11 +11,13 @@ import java.io.IOException; import java.net.URL; +import java.util.Locale; /** * This is a Threat Intel Source config where the iocs are downloaded from the URL */ public class UrlDownloadSource extends Source implements Writeable, ToXContent { + private static final Logger log = LogManager.getLogger(UrlDownloadSource.class); public static final String URL_FIELD = "url"; public static final String FEED_FORMAT_FIELD = "feed_format"; public static final String HAS_CSV_HEADER_FIELD = "has_csv_header_field"; @@ -71,6 +75,11 @@ public static UrlDownloadSource parse(XContentParser xcp) throws IOException { case URL_FIELD: String urlString = xcp.text(); url = new URL(urlString); + String protocol = url.getProtocol().toLowerCase(Locale.ROOT); + if (!"http".equals(protocol) && !"https".equals(protocol)) { + log.error("Unsupported protocol [{}]. Only http and https are allowed. Url:{}", protocol, urlString); + throw new IOException("Unsupported protocol [" + protocol + "]. Only http and https are allowed."); + } break; case FEED_FORMAT_FIELD: feedFormat = xcp.text(); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java index 3cbf31086..f85979c99 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java @@ -17,10 +17,13 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; +import java.net.InetAddress; import java.net.URL; import java.net.URLConnection; +import java.net.UnknownHostException; import java.security.AccessController; import java.security.PrivilegedAction; +import java.util.Locale; //Parser helper class public class ThreatIntelFeedParser { @@ -34,6 +37,11 @@ public class ThreatIntelFeedParser { */ @SuppressForbidden(reason = "Need to connect to http endpoint to read threat intel feed database file") public static CSVParser getThreatIntelFeedReaderCSV(final TIFMetadata tifMetadata) { + try { + validateUrl(new URL(tifMetadata.getUrl())); + } catch (IOException e) { + throw new OpenSearchException("Invalid threat intel feed URL [{}]", tifMetadata.getUrl(), e); + } SpecialPermission.check(); return AccessController.doPrivileged((PrivilegedAction) () -> { try { @@ -53,6 +61,7 @@ public static CSVParser getThreatIntelFeedReaderCSV(final TIFMetadata tifMetadat */ @SuppressForbidden(reason = "Need to connect to http endpoint to read threat intel feed database file") public static CSVParser getThreatIntelFeedReaderCSV(URL url) { + validateUrl(url); SpecialPermission.check(); return AccessController.doPrivileged((PrivilegedAction) () -> { try { @@ -65,4 +74,32 @@ public static CSVParser getThreatIntelFeedReaderCSV(URL url) { } }); } + + private static void validateUrl(URL url) { + String protocol = url.getProtocol().toLowerCase(Locale.ROOT); + if (!"http".equals(protocol) && !"https".equals(protocol)) { + log.error("Unsupported protocol [{}]. Only http and https are allowed.", protocol); + throw new OpenSearchException("Unsupported protocol [{}]. Only http and https are allowed.", protocol); + } + + InetAddress address; + try { + address = InetAddress.getByName(url.getHost()); + } catch (UnknownHostException e) { + log.error("Unable to resolve host [{}]", url.getHost()); + throw new OpenSearchException("Unable to resolve host [{}]", url.getHost()); + } + + if (address.isLoopbackAddress() + || address.isLinkLocalAddress() + || address.isSiteLocalAddress() + || address.isAnyLocalAddress()) { + log.error("URL [{}] points to a restricted address. Loopback, link-local, and private addresses are not allowed.", + url); + throw new OpenSearchException( + "URL [{}] points to a restricted address. Loopback, link-local, and private addresses are not allowed.", + url + ); + } + } } diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/model/ThreatIntelSourceTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/model/ThreatIntelSourceTests.java index dd6c13b07..587711fdb 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/model/ThreatIntelSourceTests.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/model/ThreatIntelSourceTests.java @@ -5,6 +5,7 @@ package org.opensearch.securityanalytics.threatIntel.model; +import org.apache.commons.lang3.tuple.Pair; import org.junit.Test; import org.opensearch.core.rest.RestStatus; import org.opensearch.securityanalytics.TestHelpers; @@ -70,4 +71,26 @@ public void testParseInvalidSourceField() { assertEquals(RestStatus.BAD_REQUEST, exception.status()); assertTrue(exception.getMessage().contains("Unexpected input in 'source' field when reading ioc store config.")); } + + @Test + public void testParseWithUrlDownloadSource_fileProtocolBlocked() { + Pair[] blockedUrls = new Pair[] { + Pair.of("file:///etc/passwd", "file"), + Pair.of("ftp://example.com/feed.csv", "ftp"), + Pair.of("jar:file:///tmp/test.jar!/", "jar") + }; + + for (Pair blockedUrl : blockedUrls) { + String sourceString = "{\n" + + " \"url_download\": {\n" + + " \"url\": \"" + blockedUrl.getLeft() + "\",\n" + + " \"feed_format\": \"csv\"\n" + + " }\n" + + "}"; + Exception e = assertThrows(IOException.class, + () -> Source.parse(TestHelpers.parser(sourceString))); + assertEquals(String.format("Unsupported protocol [%s]. Only http and https are allowed.", blockedUrl.getRight()), + e.getMessage()); + } + } } diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParserTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParserTests.java new file mode 100644 index 000000000..8ad0eb571 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParserTests.java @@ -0,0 +1,52 @@ +package org.opensearch.securityanalytics.threatIntel.util; + +import org.junit.Test; +import org.opensearch.OpenSearchException; +import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; +import org.opensearch.test.OpenSearchTestCase; + +import java.net.URL; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class ThreatIntelFeedParserTests extends OpenSearchTestCase { + + @Test + public void testGetThreatIntelFeedReaderCSV_blockedUrls() { + String[] blockedUrls = { + "file:///etc/passwd", //fileProtocolBlocked + "ftp://example.com/feed.csv", //ftpProtocolBlocked + "http://127.0.0.1:9200", //loopbackBlocked + "http://localhost:9200", //localhostBlocked + "http://169.254.169.254/latest/meta-data/", //linkLocalBlocked + "http://10.0.0.1/feed.csv", //siteLocalBlocked + "http://192.168.1.1/feed.csv", //privateNetworkBlocked + "jar:file:///tmp/test.jar!/" //jarProtocolBlocked + }; + + for (String blockedUrl : blockedUrls) { + expectThrows(OpenSearchException.class, + () -> ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(new URL(blockedUrl))); + } + } + + @Test + public void testGetThreatIntelFeedReaderCSV_tifMetadata_blockedUrls() { + String[] blockedUrls = { + "file:///etc/passwd", + "http://127.0.0.1:9200", + "http://localhost:9200", + "http://169.254.169.254/latest/meta-data/", + "http://10.0.0.1/feed.csv", + "http://192.168.1.1/feed.csv" + }; + + for (String blockedUrl : blockedUrls) { + TIFMetadata tifMetadata = mock(TIFMetadata.class); + when(tifMetadata.getUrl()).thenReturn(blockedUrl); + expectThrows(OpenSearchException.class, + () -> ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(tifMetadata)); + } + } +}