Skip to content

Commit 8e3d6fe

Browse files
mrodmmrodm
authored
Allow new documentation ranges in the validation of IPs (#2529)
This PR adds the latest IP ranges to the isDocumentation so they can be validated successfully in elastic-package tests.
1 parent 35aeb46 commit 8e3d6fe

File tree

5 files changed

+96
-6
lines changed

5 files changed

+96
-6
lines changed

‎docs/howto/ingest_geoip.md

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,19 @@
22

33
Elasticsearch provides default GeoIP databases that can be downloaded in runtime and which weights ~70 MB. This can be
44
a root cause of flakiness of package tests, so elastic-package embeds small samples of GeoIP databases, that can identify
5-
accurately only few ranges of IP addresses included [here](../../internal/fields/_static/allowed_geo_ips.txt)
5+
accurately only few ranges of IP addresses.
6+
7+
Specifically, the following documentation ranges of IP addresses are included in those GeoIP databases:
8+
- [RFC5737](https://datatracker.ietf.org/doc/rfc5737/)
9+
- 192.0.2.0/24
10+
- 198.51.100.0/24
11+
- 203.0.113.0/24
12+
- [RFC6676](https://datatracker.ietf.org/doc/rfc6676/) (multicast addresses allocated for documentation purposes):
13+
- 233.252.0.0/24
14+
- [RFC3849](https://datatracker.ietf.org/doc/rfc3849/)
15+
- "2001:DB8::/32"
16+
- [RFC9637](https://datatracker.ietf.org/doc/rfc9637/)
17+
- "3fff::/20"
618

719
If you want the ingest pipeline to include a "geo" section in the event, feel free to use one of above IP addresses.
820
Embedded databases contain information about: cities, countries and ASNs.

‎internal/fields/validate.go

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1263,29 +1263,35 @@ func (v *Validator) parseSingleElementValue(key string, definition FieldDefiniti
12631263
}
12641264

12651265
// isDocumentation reports whether ip is a reserved address for documentation,
1266-
// according to RFC 5737 (IPv4 Address Blocks Reserved for Documentation) and
1267-
// RFC 3849 (IPv6 Address Prefix Reserved for Documentation).
1266+
// according to RFC 5737 (IPv4 Address Blocks Reserved for Documentation), RFC 6676,
1267+
// RFC 3849 (IPv6 Address Prefix Reserved for Documentation) and RFC 9637.
12681268
func isDocumentation(ip net.IP) bool {
12691269
if ip4 := ip.To4(); ip4 != nil {
12701270
// Following RFC 5737, Section 3. Documentation Address Blocks which says:
12711271
// The blocks 192.0.2.0/24 (TEST-NET-1), 198.51.100.0/24 (TEST-NET-2),
12721272
// and 203.0.113.0/24 (TEST-NET-3) are provided for use in
12731273
// documentation.
1274+
// Following RFC 6676, the IPV4 multicast addresses allocated for documentation
1275+
// purposes are 233.252.0.0/24
12741276
return ((ip4[0] == 192 && ip4[1] == 0 && ip4[2] == 2) ||
12751277
(ip4[0] == 198 && ip4[1] == 51 && ip4[2] == 100) ||
1276-
(ip4[0] == 203 && ip4[1] == 0 && ip4[2] == 113))
1278+
(ip4[0] == 203 && ip4[1] == 0 && ip4[2] == 113) ||
1279+
(ip4[0] == 233 && ip4[1] == 252 && ip4[2] == 0))
12771280
}
12781281
// Following RFC 3849, Section 2. Documentation IPv6 Address Prefix which
12791282
// says:
12801283
// The prefix allocated for documentation purposes is 2001:DB8::/32
1281-
return len(ip) == net.IPv6len && ip[0] == 32 && ip[1] == 1 && ip[2] == 13 && ip[3] == 184
1284+
// Following RFC 9637, a new address block 3fff::/20 is registered for documentation purposes
1285+
return len(ip) == net.IPv6len &&
1286+
(ip[0] == 32 && ip[1] == 1 && ip[2] == 13 && ip[3] == 184) ||
1287+
(ip[0] == 63 && ip[1] == 255 && ip[2] <= 15)
12821288
}
12831289

12841290
// isAllowedIPValue checks if the provided IP is allowed for testing
12851291
// The set of allowed IPs are:
12861292
// - private IPs as described in RFC 1918 & RFC 4193
12871293
// - public IPs allowed by MaxMind for testing
1288-
// - Reserved IPs for documentation RFC 5737 and RFC 3849
1294+
// - Reserved IPs for documentation RFC 5737, RFC 3849, RFC 6676 and RFC 9637
12891295
// - 0.0.0.0 and 255.255.255.255 for IPv4
12901296
// - 0:0:0:0:0:0:0:0 and ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff for IPv6
12911297
func (v *Validator) isAllowedIPValue(s string) bool {

‎internal/fields/validate_test.go

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1247,6 +1247,24 @@ func Test_IsAllowedIPValue(t *testing.T) {
12471247
"89.160.20.112/28",
12481248
},
12491249
},
1250+
{
1251+
title: "valid ipv4 multicast address",
1252+
ip: "233.252.0.57",
1253+
expected: true,
1254+
allowedIps: []string{},
1255+
},
1256+
{
1257+
title: "second range documentation ipv6",
1258+
ip: "3fff:0000:0000:0000:0000:1000:1000:1000",
1259+
expected: true,
1260+
allowedIps: []string{},
1261+
},
1262+
{
1263+
title: "other invalid ipv6",
1264+
ip: "3fff:1fff:ffff:ffff:ffff:ffff:ffff:ffff",
1265+
expected: false,
1266+
allowedIps: []string{},
1267+
},
12501268
}
12511269

12521270
for _, c := range cases {

‎test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
::1 - - [26/Dec/2016:16:16:28 +0200] "GET / HTTP/1.1" 200 45
22
::1 - - [26/Dec/2016:16:16:29 +0200] "GET /favicon.ico HTTP/1.1" 404 209
33
::1 - - [26/Dec/2016:16:16:48 +0200] "-" 408 -
4+
3fff::1 - - [26/Dec/2016:16:16:48 +0200] "-" 408 -
45
89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] "GET / HTTP/1.1" 200 45
56
89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] "GET /notfound HTTP/1.1" 404 206
67
89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] "GET /hmm HTTP/1.1" 404 201

‎test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,59 @@
134134
"name": "-"
135135
}
136136
},
137+
{
138+
"@timestamp": "2016-12-26T14:16:48.000Z",
139+
"apache": {
140+
"access": {}
141+
},
142+
"ecs": {
143+
"version": "1.12.0"
144+
},
145+
"event": {
146+
"category": [
147+
"web"
148+
],
149+
"created": "2020-04-28T11:07:58.223Z",
150+
"ingested": "2022-09-08T10:05:05.950497950Z",
151+
"kind": "event",
152+
"original": "3fff::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -",
153+
"outcome": "failure",
154+
"type": [
155+
"access"
156+
]
157+
},
158+
"http": {
159+
"response": {
160+
"status_code": 408
161+
}
162+
},
163+
"source": {
164+
"address": "3fff::1",
165+
"as": {
166+
"number": 65552,
167+
"organization": {
168+
"name": "Documentation ASN"
169+
}
170+
},
171+
"geo": {
172+
"city_name": "Lisbon",
173+
"continent_name": "Europe",
174+
"country_iso_code": "PT",
175+
"country_name": "Portugal",
176+
"location": {
177+
"lat": 38.71667,
178+
"lon": -9.13333
179+
}
180+
},
181+
"ip": "3fff::1"
182+
},
183+
"tags": [
184+
"preserve_original_event"
185+
],
186+
"user": {
187+
"name": "-"
188+
}
189+
},
137190
{
138191
"@timestamp": "2016-12-26T16:23:35.000Z",
139192
"apache": {

0 commit comments

Comments
 (0)