Skip to content

Commit 28d4cb0

Browse files
nishtham-amazonnishtham-amazon
committed
Add validate url check while reading thread intel feed (opensearch-project#1660)
* Add validate url check while reading thread intel feed Signed-off-by: nishtham <nishtham@amazon.com> * Added unit tests for url validation for threat intel feeds --------- Signed-off-by: nishtham <nishtham@amazon.com>
1 parent 55384b3 commit 28d4cb0

File tree

4 files changed

+121
-0
lines changed

4 files changed

+121
-0
lines changed

‎src/main/java/org/opensearch/securityanalytics/threatIntel/model/UrlDownloadSource.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package org.opensearch.securityanalytics.threatIntel.model;
22

3+
import org.apache.logging.log4j.LogManager;
4+
import org.apache.logging.log4j.Logger;
35
import org.opensearch.core.common.io.stream.StreamInput;
46
import org.opensearch.core.common.io.stream.StreamOutput;
57
import org.opensearch.core.common.io.stream.Writeable;
@@ -9,11 +11,13 @@
911

1012
import java.io.IOException;
1113
import java.net.URL;
14+
import java.util.Locale;
1215

1316
/**
1417
* This is a Threat Intel Source config where the iocs are downloaded from the URL
1518
*/
1619
public class UrlDownloadSource extends Source implements Writeable, ToXContent {
20+
private static final Logger log = LogManager.getLogger(UrlDownloadSource.class);
1721
public static final String URL_FIELD = "url";
1822
public static final String FEED_FORMAT_FIELD = "feed_format";
1923
public static final String HAS_CSV_HEADER_FIELD = "has_csv_header_field";
@@ -71,6 +75,11 @@ public static UrlDownloadSource parse(XContentParser xcp) throws IOException {
7175
case URL_FIELD:
7276
String urlString = xcp.text();
7377
url = new URL(urlString);
78+
String protocol = url.getProtocol().toLowerCase(Locale.ROOT);
79+
if (!"http".equals(protocol) && !"https".equals(protocol)) {
80+
log.error("Unsupported protocol [{}]. Only http and https are allowed. Url:{}", protocol, urlString);
81+
throw new IOException("Unsupported protocol [" + protocol + "]. Only http and https are allowed.");
82+
}
7483
break;
7584
case FEED_FORMAT_FIELD:
7685
feedFormat = xcp.text();

‎src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,13 @@
1717
import java.io.BufferedReader;
1818
import java.io.IOException;
1919
import java.io.InputStreamReader;
20+
import java.net.InetAddress;
2021
import java.net.URL;
2122
import java.net.URLConnection;
23+
import java.net.UnknownHostException;
2224
import java.security.AccessController;
2325
import java.security.PrivilegedAction;
26+
import java.util.Locale;
2427

2528
//Parser helper class
2629
public class ThreatIntelFeedParser {
@@ -34,6 +37,11 @@ public class ThreatIntelFeedParser {
3437
*/
3538
@SuppressForbidden(reason = "Need to connect to http endpoint to read threat intel feed database file")
3639
public static CSVParser getThreatIntelFeedReaderCSV(final TIFMetadata tifMetadata) {
40+
try {
41+
validateUrl(new URL(tifMetadata.getUrl()));
42+
} catch (IOException e) {
43+
throw new OpenSearchException("Invalid threat intel feed URL [{}]", tifMetadata.getUrl(), e);
44+
}
3745
SpecialPermission.check();
3846
return AccessController.doPrivileged((PrivilegedAction<CSVParser>) () -> {
3947
try {
@@ -53,6 +61,7 @@ public static CSVParser getThreatIntelFeedReaderCSV(final TIFMetadata tifMetadat
5361
*/
5462
@SuppressForbidden(reason = "Need to connect to http endpoint to read threat intel feed database file")
5563
public static CSVParser getThreatIntelFeedReaderCSV(URL url) {
64+
validateUrl(url);
5665
SpecialPermission.check();
5766
return AccessController.doPrivileged((PrivilegedAction<CSVParser>) () -> {
5867
try {
@@ -65,4 +74,32 @@ public static CSVParser getThreatIntelFeedReaderCSV(URL url) {
6574
}
6675
});
6776
}
77+
78+
private static void validateUrl(URL url) {
79+
String protocol = url.getProtocol().toLowerCase(Locale.ROOT);
80+
if (!"http".equals(protocol) && !"https".equals(protocol)) {
81+
log.error("Unsupported protocol [{}]. Only http and https are allowed.", protocol);
82+
throw new OpenSearchException("Unsupported protocol [{}]. Only http and https are allowed.", protocol);
83+
}
84+
85+
InetAddress address;
86+
try {
87+
address = InetAddress.getByName(url.getHost());
88+
} catch (UnknownHostException e) {
89+
log.error("Unable to resolve host [{}]", url.getHost());
90+
throw new OpenSearchException("Unable to resolve host [{}]", url.getHost());
91+
}
92+
93+
if (address.isLoopbackAddress()
94+
|| address.isLinkLocalAddress()
95+
|| address.isSiteLocalAddress()
96+
|| address.isAnyLocalAddress()) {
97+
log.error("URL [{}] points to a restricted address. Loopback, link-local, and private addresses are not allowed.",
98+
url);
99+
throw new OpenSearchException(
100+
"URL [{}] points to a restricted address. Loopback, link-local, and private addresses are not allowed.",
101+
url
102+
);
103+
}
104+
}
68105
}

‎src/test/java/org/opensearch/securityanalytics/threatIntel/model/ThreatIntelSourceTests.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
package org.opensearch.securityanalytics.threatIntel.model;
77

8+
import org.apache.commons.lang3.tuple.Pair;
89
import org.junit.Test;
910
import org.opensearch.core.rest.RestStatus;
1011
import org.opensearch.securityanalytics.TestHelpers;
@@ -70,4 +71,26 @@ public void testParseInvalidSourceField() {
7071
assertEquals(RestStatus.BAD_REQUEST, exception.status());
7172
assertTrue(exception.getMessage().contains("Unexpected input in 'source' field when reading ioc store config."));
7273
}
74+
75+
@Test
76+
public void testParseWithUrlDownloadSource_fileProtocolBlocked() {
77+
Pair<String, String>[] blockedUrls = new Pair[] {
78+
Pair.of("file:///etc/passwd", "file"),
79+
Pair.of("ftp://example.com/feed.csv", "ftp"),
80+
Pair.of("jar:file:///tmp/test.jar!/", "jar")
81+
};
82+
83+
for (Pair<String, String> blockedUrl : blockedUrls) {
84+
String sourceString = "{\n" +
85+
" \"url_download\": {\n" +
86+
" \"url\": \"" + blockedUrl.getLeft() + "\",\n" +
87+
" \"feed_format\": \"csv\"\n" +
88+
" }\n" +
89+
"}";
90+
Exception e = assertThrows(IOException.class,
91+
() -> Source.parse(TestHelpers.parser(sourceString)));
92+
assertEquals(String.format("Unsupported protocol [%s]. Only http and https are allowed.", blockedUrl.getRight()),
93+
e.getMessage());
94+
}
95+
}
7396
}

‎src/test/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParserTests.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
package org.opensearch.securityanalytics.threatIntel.util;
2+
3+
import org.junit.Test;
4+
import org.opensearch.OpenSearchException;
5+
import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata;
6+
import org.opensearch.test.OpenSearchTestCase;
7+
8+
import java.net.URL;
9+
10+
import static org.mockito.Mockito.mock;
11+
import static org.mockito.Mockito.when;
12+
13+
public class ThreatIntelFeedParserTests extends OpenSearchTestCase {
14+
15+
@Test
16+
public void testGetThreatIntelFeedReaderCSV_blockedUrls() {
17+
String[] blockedUrls = {
18+
"file:///etc/passwd", //fileProtocolBlocked
19+
"ftp://example.com/feed.csv", //ftpProtocolBlocked
20+
"http://127.0.0.1:9200", //loopbackBlocked
21+
"http://localhost:9200", //localhostBlocked
22+
"http://169.254.169.254/latest/meta-data/", //linkLocalBlocked
23+
"http://10.0.0.1/feed.csv", //siteLocalBlocked
24+
"http://192.168.1.1/feed.csv", //privateNetworkBlocked
25+
"jar:file:///tmp/test.jar!/" //jarProtocolBlocked
26+
};
27+
28+
for (String blockedUrl : blockedUrls) {
29+
expectThrows(OpenSearchException.class,
30+
() -> ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(new URL(blockedUrl)));
31+
}
32+
}
33+
34+
@Test
35+
public void testGetThreatIntelFeedReaderCSV_tifMetadata_blockedUrls() {
36+
String[] blockedUrls = {
37+
"file:///etc/passwd",
38+
"http://127.0.0.1:9200",
39+
"http://localhost:9200",
40+
"http://169.254.169.254/latest/meta-data/",
41+
"http://10.0.0.1/feed.csv",
42+
"http://192.168.1.1/feed.csv"
43+
};
44+
45+
for (String blockedUrl : blockedUrls) {
46+
TIFMetadata tifMetadata = mock(TIFMetadata.class);
47+
when(tifMetadata.getUrl()).thenReturn(blockedUrl);
48+
expectThrows(OpenSearchException.class,
49+
() -> ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(tifMetadata));
50+
}
51+
}
52+
}

0 commit comments

Comments
 (0)