diff --git a/apidef/api_definitions.go b/apidef/api_definitions.go index 1f11a4fa66d..c525ffd343b 100644 --- a/apidef/api_definitions.go +++ b/apidef/api_definitions.go @@ -595,22 +595,6 @@ type ResponseProcessor struct { Options interface{} `bson:"options" json:"options"` } -type HostCheckObject struct { - CheckURL string `bson:"url" json:"url"` - Protocol string `bson:"protocol" json:"protocol"` - Timeout time.Duration `bson:"timeout" json:"timeout"` - EnableProxyProtocol bool `bson:"enable_proxy_protocol" json:"enable_proxy_protocol"` - Commands []CheckCommand `bson:"commands" json:"commands"` - Method string `bson:"method" json:"method"` - Headers map[string]string `bson:"headers" json:"headers"` - Body string `bson:"body" json:"body"` -} - -type CheckCommand struct { - Name string `bson:"name" json:"name"` - Message string `bson:"message" json:"message"` -} - type ServiceDiscoveryConfiguration struct { UseDiscoveryService bool `bson:"use_discovery_service" json:"use_discovery_service"` QueryEndpoint string `bson:"query_endpoint" json:"query_endpoint"` @@ -903,11 +887,40 @@ type AnalyticsPluginConfig struct { FuncName string `bson:"func_name" json:"func_name,omitempty"` } +// UptimeTests holds the test configuration for uptime tests. type UptimeTests struct { CheckList []HostCheckObject `bson:"check_list" json:"check_list"` Config UptimeTestsConfig `bson:"config" json:"config"` } +// UptimeTestCommand handles additional checks for tcp connections. +type CheckCommand struct { + Name string `bson:"name" json:"name"` + Message string `bson:"message" json:"message"` +} + +// HostCheckObject represents a single uptime test check. +type HostCheckObject struct { + CheckURL string `bson:"url" json:"url"` + Protocol string `bson:"protocol" json:"protocol"` + Timeout time.Duration `bson:"timeout" json:"timeout"` + EnableProxyProtocol bool `bson:"enable_proxy_protocol" json:"enable_proxy_protocol"` + Commands []CheckCommand `bson:"commands" json:"commands"` + Method string `bson:"method" json:"method"` + Headers map[string]string `bson:"headers" json:"headers"` + Body string `bson:"body" json:"body"` +} + +// AddCommand will append a new command to the test. +func (h *HostCheckObject) AddCommand(name, message string) { + command := CheckCommand{ + Name: name, + Message: message, + } + + h.Commands = append(h.Commands, command) +} + type UptimeTestsConfig struct { ExpireUptimeAnalyticsAfter int64 `bson:"expire_utime_after" json:"expire_utime_after"` // must have an expireAt TTL index set (http://docs.mongodb.org/manual/tutorial/expire-data/) ServiceDiscovery ServiceDiscoveryConfiguration `bson:"service_discovery" json:"service_discovery"` diff --git a/apidef/oas/Taskfile.yml b/apidef/oas/Taskfile.yml index 0788431727e..17b0f674ac2 100644 --- a/apidef/oas/Taskfile.yml +++ b/apidef/oas/Taskfile.yml @@ -11,6 +11,7 @@ tasks: test: desc: "Run tests" cmds: + - cd schema && ./build.sh && cd - - go fmt ./... - go test -count=1 ./... diff --git a/apidef/oas/linter_test.go b/apidef/oas/linter_test.go index d838a8f1b8b..9c71e8cf75f 100644 --- a/apidef/oas/linter_test.go +++ b/apidef/oas/linter_test.go @@ -99,6 +99,29 @@ func TestXTykGateway_Lint(t *testing.T) { OAuth: nil, } + settings.Upstream.UptimeTests = &UptimeTests{ + HostDownRetestPeriod: ReadableDuration(10 * time.Second), + LogRetentionPeriod: ReadableDuration(10 * time.Second), + Tests: []UptimeTest{ + { + Timeout: ReadableDuration(10 * time.Second), + Commands: []UptimeTestCommand{ + { + Name: "send", + Message: "PING", + }, + { + Name: "recv", + Message: "+PONG", + }, + }, + Headers: map[string]string{ + "Request-Id": "1", + }, + }, + }, + } + settings.Upstream.TLSTransport.MinVersion = "1.2" settings.Upstream.TLSTransport.MaxVersion = "1.2" settings.Upstream.TLSTransport.Ciphers = []string{"TLS_RSA_WITH_RC4_128_SHA"} diff --git a/apidef/oas/oas_test.go b/apidef/oas/oas_test.go index a684216da34..30e8dbc7553 100644 --- a/apidef/oas/oas_test.go +++ b/apidef/oas/oas_test.go @@ -221,18 +221,7 @@ func TestOAS_ExtractTo_ResetAPIDefinition(t *testing.T) { "APIDefinition.VersionData.Versions[0].ExtendedPaths.PersistGraphQL[0].Operation", "APIDefinition.VersionData.Versions[0].ExtendedPaths.PersistGraphQL[0].Variables[0]", "APIDefinition.VersionData.Versions[0].IgnoreEndpointCase", - "APIDefinition.UptimeTests.CheckList[0].CheckURL", - "APIDefinition.UptimeTests.CheckList[0].Protocol", - "APIDefinition.UptimeTests.CheckList[0].Timeout", - "APIDefinition.UptimeTests.CheckList[0].EnableProxyProtocol", - "APIDefinition.UptimeTests.CheckList[0].Commands[0].Name", - "APIDefinition.UptimeTests.CheckList[0].Commands[0].Message", - "APIDefinition.UptimeTests.CheckList[0].Method", - "APIDefinition.UptimeTests.CheckList[0].Headers[0]", - "APIDefinition.UptimeTests.CheckList[0].Body", - "APIDefinition.UptimeTests.Config.ExpireUptimeAnalyticsAfter", "APIDefinition.UptimeTests.Config.ServiceDiscovery.CacheDisabled", - "APIDefinition.UptimeTests.Config.RecheckWait", "APIDefinition.Proxy.DisableStripSlash", "APIDefinition.Proxy.CheckHostAgainstUptimeTests", "APIDefinition.DisableQuota", diff --git a/apidef/oas/schema/x-tyk-api-gateway.json b/apidef/oas/schema/x-tyk-api-gateway.json index b90a327251a..978be9ee3b2 100644 --- a/apidef/oas/schema/x-tyk-api-gateway.json +++ b/apidef/oas/schema/x-tyk-api-gateway.json @@ -1341,13 +1341,8 @@ "serviceDiscovery": { "$ref": "#/definitions/X-Tyk-ServiceDiscovery" }, - "test": { - "type": "object", - "properties": { - "serviceDiscovery": { - "$ref": "#/definitions/X-Tyk-ServiceDiscovery" - } - } + "uptimeTests": { + "$ref": "#/definitions/X-Tyk-UptimeTests" }, "mutualTLS": { "$ref": "#/definitions/X-Tyk-MutualTLS" @@ -2405,14 +2400,84 @@ }, "X-Tyk-PreserveHostHeader": { "type": "object", - "properties": { - "enabled": { - "type": "boolean" - } + "properties": { + "enabled": { + "type": "boolean" + } + }, + "required": [ + "enabled" + ] + }, + "X-Tyk-UptimeTestCommand": { + "type": "object", + "properties": { + "message": { + "type": "string" }, - "required": [ - "enabled" - ] + "name": { + "type": "string" + } + } + }, + "X-Tyk-UptimeTest": { + "type": "object", + "properties": { + "body": { + "type": "string" + }, + "commands": { + "type": "array", + "items": { + "$ref": "#/definitions/X-Tyk-UptimeTestCommand" + } + }, + "enableProxyProtocol": { + "type": "boolean" + }, + "headers": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "method": { + "type": "string" + }, + "protocol": { + "type": "string" + }, + "timeout": { + "$ref": "#/definitions/X-Tyk-ReadableDuration" + }, + "url": { + "type": "string" + } + } + }, + "X-Tyk-UptimeTests": { + "type": "object", + "properties": { + "hostDownRetestPeriod": { + "$ref": "#/definitions/X-Tyk-ReadableDuration" + }, + "logRetentionPeriod": { + "$ref": "#/definitions/X-Tyk-ReadableDuration" + }, + "serviceDiscovery": { + "$ref": "#/definitions/X-Tyk-ServiceDiscovery" + }, + "tests": { + "type": "array", + "items": { + "$ref": "#/definitions/X-Tyk-UptimeTest" + } + } + } + }, + "X-Tyk-ReadableDuration": { + "type": "string", + "pattern": "^(\\d+h)?(\\d+m)?(\\d+s)?$" }, "X-Tyk-LoadBalancingTarget": { "type": "object", diff --git a/apidef/oas/schema/x-tyk-api-gateway.strict.json b/apidef/oas/schema/x-tyk-api-gateway.strict.json index 6ec5b774853..4b5c7a61a90 100644 --- a/apidef/oas/schema/x-tyk-api-gateway.strict.json +++ b/apidef/oas/schema/x-tyk-api-gateway.strict.json @@ -1394,13 +1394,8 @@ "serviceDiscovery": { "$ref": "#/definitions/X-Tyk-ServiceDiscovery" }, - "test": { - "type": "object", - "properties": { - "serviceDiscovery": { - "$ref": "#/definitions/X-Tyk-ServiceDiscovery" - } - } + "uptimeTests": { + "$ref": "#/definitions/X-Tyk-UptimeTests" }, "mutualTLS": { "$ref": "#/definitions/X-Tyk-MutualTLS" @@ -1419,10 +1414,25 @@ }, "preserveHostHeader": { "$ref": "#/definitions/X-Tyk-PreserveHostHeader" + }, + "tlsTransport": { + "$ref": "#/definitions/X-Tyk-TLSTransport" + }, + "proxy": { + "$ref": "#/definitions/X-Tyk-Proxy" } }, - "required": [ - "url" + "anyOf": [ + { + "required": [ + "url" + ] + }, + { + "required": [ + "loadBalancing" + ] + } ], "additionalProperties": false }, @@ -2195,6 +2205,9 @@ }, "oauth": { "$ref": "#/definitions/X-Tyk-UpstreamOAuth" + }, + "requestSigning": { + "$ref": "#/definitions/X-Tyk-UpstreamRequestSigning" } }, "required": [ @@ -2327,6 +2340,39 @@ ], "additionalProperties": false }, + "X-Tyk-UpstreamRequestSigning": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "signatureHeader": { + "type": "string" + }, + "algorithm": { + "type": "string" + }, + "keyId": { + "type": "string" + }, + "headers": { + "type": "array", + "items": { + "type": "string" + } + }, + "secret": { + "type": "string" + }, + "certificateId": { + "type": "string" + } + }, + "required": [ + "enabled" + ], + "additionalProperties": false + }, "X-Tyk-NonEmptyString": { "type": "string", "pattern": "\\S+", @@ -2394,6 +2440,78 @@ "required": [ "enabled" ], + "allOf": [ + { + "if": { + "properties": { + "enabled": { + "const": true + } + } + }, + "then": { + "properties": { + "targets": { + "minItems": 1 + } + }, + "required": [ + "targets" + ] + } + } + ], + "additionalProperties": false + }, + "X-Tyk-TLSTransport": { + "type": "object", + "properties": { + "insecureSkipVerify": { + "type": "boolean" + }, + "ciphers": { + "type": "array", + "items": { + "type": "string" + } + }, + "minVersion": { + "type": "string", + "enum": [ + "1.0", + "1.1", + "1.2", + "1.3" + ] + }, + "maxVersion": { + "type": "string", + "enum": [ + "1.0", + "1.1", + "1.2", + "1.3" + ] + }, + "forceCommonNameCheck": { + "type": "boolean" + } + }, + "additionalProperties": false + }, + "X-Tyk-Proxy": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "url": { + "type": "string" + } + }, + "required": [ + "enabled" + ], "additionalProperties": false }, "X-Tyk-PreserveHostHeader": { @@ -2408,6 +2526,80 @@ ], "additionalProperties": false }, + "X-Tyk-UptimeTestCommand": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "name": { + "type": "string" + } + }, + "additionalProperties": false + }, + "X-Tyk-UptimeTest": { + "type": "object", + "properties": { + "body": { + "type": "string" + }, + "commands": { + "type": "array", + "items": { + "$ref": "#/definitions/X-Tyk-UptimeTestCommand" + } + }, + "enable_proxy_protocol": { + "type": "boolean" + }, + "headers": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "method": { + "type": "string" + }, + "protocol": { + "type": "string" + }, + "timeout": { + "$ref": "#/definitions/X-Tyk-ReadableDuration" + }, + "url": { + "type": "string" + } + }, + "additionalProperties": false + }, + "X-Tyk-UptimeTests": { + "type": "object", + "properties": { + "hostDownRetestPeriod": { + "$ref": "#/definitions/X-Tyk-ReadableDuration" + }, + "logRetentionPeriod": { + "$ref": "#/definitions/X-Tyk-ReadableDuration" + }, + "serviceDiscovery": { + "$ref": "#/definitions/X-Tyk-ServiceDiscovery" + }, + "tests": { + "type": "array", + "items": { + "$ref": "#/definitions/X-Tyk-UptimeTest" + } + } + }, + "additionalProperties": false + }, + "X-Tyk-ReadableDuration": { + "type": "string", + "pattern": "^(\\d+h)?(\\d+m)?(\\d+s)?$", + "additionalProperties": false + }, "X-Tyk-LoadBalancingTarget": { "type": "object", "properties": { diff --git a/apidef/oas/upstream.go b/apidef/oas/upstream.go index 5afc3ca00a5..a0f39e9b40f 100644 --- a/apidef/oas/upstream.go +++ b/apidef/oas/upstream.go @@ -19,8 +19,8 @@ type Upstream struct { // Tyk classic API definition: `proxy.service_discovery` ServiceDiscovery *ServiceDiscovery `bson:"serviceDiscovery,omitempty" json:"serviceDiscovery,omitempty"` - // Test contains the configuration related to uptime tests. - Test *Test `bson:"test,omitempty" json:"test,omitempty"` + // UptimeTests contains the configuration related to uptime tests. + UptimeTests *UptimeTests `bson:"uptimeTests,omitempty" json:"uptimeTests,omitempty"` // MutualTLS contains the configuration for establishing a mutual TLS connection between Tyk and the upstream server. MutualTLS *MutualTLS `bson:"mutualTLS,omitempty" json:"mutualTLS,omitempty"` @@ -61,13 +61,13 @@ func (u *Upstream) Fill(api apidef.APIDefinition) { u.ServiceDiscovery = nil } - if u.Test == nil { - u.Test = &Test{} + if u.UptimeTests == nil { + u.UptimeTests = &UptimeTests{} } - u.Test.Fill(api.UptimeTests) - if ShouldOmit(u.Test) { - u.Test = nil + u.UptimeTests.Fill(api.UptimeTests) + if ShouldOmit(u.UptimeTests) { + u.UptimeTests = nil } if u.MutualTLS == nil { @@ -151,14 +151,14 @@ func (u *Upstream) ExtractTo(api *apidef.APIDefinition) { u.ServiceDiscovery.ExtractTo(&api.Proxy.ServiceDiscovery) - if u.Test == nil { - u.Test = &Test{} + if u.UptimeTests == nil { + u.UptimeTests = &UptimeTests{} defer func() { - u.Test = nil + u.UptimeTests = nil }() } - u.Test.ExtractTo(&api.UptimeTests) + u.UptimeTests.ExtractTo(&api.UptimeTests) if u.MutualTLS == nil { u.MutualTLS = &MutualTLS{} @@ -534,15 +534,84 @@ func (sd *ServiceDiscovery) ExtractTo(serviceDiscovery *apidef.ServiceDiscoveryC serviceDiscovery.CacheDisabled = !enabled } -// Test holds the test configuration for service discovery. -type Test struct { +// UptimeTests configures uptime tests. +type UptimeTests struct { // ServiceDiscovery contains the configuration related to test Service Discovery. // Tyk classic API definition: `proxy.service_discovery` ServiceDiscovery *ServiceDiscovery `bson:"serviceDiscovery,omitempty" json:"serviceDiscovery,omitempty"` + + // Tests contains individual connectivity tests defined for checking if a service is online. + Tests []UptimeTest `bson:"tests,omitempty" json:"tests,omitempty"` + + // HostDownRetestPeriod is the time to wait until rechecking a failed test. + // If undefined, the default testing interval (10s) is in use. + // Setting this to a lower value would result in quicker recovery on failed checks. + HostDownRetestPeriod time.ReadableDuration `bson:"hostDownRetestPeriod" json:"hostDownRetestPeriod"` + + // LogRetentionPeriod holds a time to live for the uptime test results. + // If unset, a value of 100 years is the default. + LogRetentionPeriod time.ReadableDuration `bson:"logRetentionPeriod" json:"logRetentionPeriod"` +} + +// UptimeTest configures an uptime test check. +type UptimeTest struct { + // CheckURL is the URL for a request. If service discovery is in use, + // the hostname will be resolved to a service host. + // + // Examples: + // + // - `http://database1.company.local` + // - `https://webcluster.service/health` + // - `127.0.0.1:6379` (for TCP checks). + CheckURL string `bson:"url" json:"url"` + + // Protocol is the protocol for the request. Supported values are + // `http` and `tcp`, depending on what kind of check is performed. + Protocol string `bson:"protocol" json:"protocol"` + + // Timeout declares a timeout for the request. If the test exceeds + // this timeout, the check fails. + Timeout time.ReadableDuration `bson:"timeout" json:"timeout"` + + // Method allows you to customize the HTTP method for the test (`GET`, `POST`,...). + Method string `bson:"method" json:"method"` + + // Headers contain any custom headers for the back end service. + Headers map[string]string `bson:"headers" json:"headers,omitempty"` + + // Body is the body of the test request. + Body string `bson:"body" json:"body"` + + // Commands are used for TCP checks. + Commands []UptimeTestCommand `bson:"commands" json:"commands,omitempty"` + + // EnableProxyProtocol enables proxy protocol support when making request. + // The back end service needs to support this. + EnableProxyProtocol bool `bson:"enableProxyProtocol" json:"enableProxyProtocol"` +} + +// AddCommand will append a new command to the test. +func (t *UptimeTest) AddCommand(name, message string) { + command := UptimeTestCommand{ + Name: name, + Message: message, + } + + t.Commands = append(t.Commands, command) +} + +// UptimeTestCommand handles additional checks for tcp connections. +type UptimeTestCommand struct { + // Name can be either `send` or `expect`, designating if the + // message should be sent, or read from the connection. + Name string `bson:"name" json:"name"` + + // Message contains the payload to send or expect. + Message string `bson:"message" json:"message"` } -// Fill fills *Test from apidef.UptimeTests. -func (t *Test) Fill(uptimeTests apidef.UptimeTests) { +// Fill fills *UptimeTests from apidef.UptimeTests. +func (t *UptimeTests) Fill(uptimeTests apidef.UptimeTests) { if t.ServiceDiscovery == nil { t.ServiceDiscovery = &ServiceDiscovery{} } @@ -551,10 +620,35 @@ func (t *Test) Fill(uptimeTests apidef.UptimeTests) { if ShouldOmit(t.ServiceDiscovery) { t.ServiceDiscovery = nil } + + t.Tests = nil + t.LogRetentionPeriod = ReadableDuration(time.Duration(uptimeTests.Config.ExpireUptimeAnalyticsAfter) * time.Second) + t.HostDownRetestPeriod = ReadableDuration(time.Duration(uptimeTests.Config.RecheckWait) * time.Second) + + result := []UptimeTest{} + for _, v := range uptimeTests.CheckList { + check := UptimeTest{ + CheckURL: v.CheckURL, + Protocol: v.Protocol, + Timeout: ReadableDuration(v.Timeout), + Method: v.Method, + Headers: v.Headers, + Body: v.Body, + } + for _, command := range v.Commands { + check.AddCommand(command.Name, command.Message) + } + + result = append(result, check) + } + + if len(result) > 0 { + t.Tests = result + } } -// ExtractTo extracts *Test into *apidef.UptimeTests. -func (t *Test) ExtractTo(uptimeTests *apidef.UptimeTests) { +// ExtractTo extracts *UptimeTests into *apidef.UptimeTests. +func (t *UptimeTests) ExtractTo(uptimeTests *apidef.UptimeTests) { if t.ServiceDiscovery == nil { t.ServiceDiscovery = &ServiceDiscovery{} defer func() { @@ -563,6 +657,32 @@ func (t *Test) ExtractTo(uptimeTests *apidef.UptimeTests) { } t.ServiceDiscovery.ExtractTo(&uptimeTests.Config.ServiceDiscovery) + + uptimeTests.Config.ExpireUptimeAnalyticsAfter = int64(t.LogRetentionPeriod.Seconds()) + uptimeTests.Config.RecheckWait = int(t.HostDownRetestPeriod.Seconds()) + + uptimeTests.CheckList = nil + + result := []apidef.HostCheckObject{} + for _, v := range t.Tests { + check := apidef.HostCheckObject{ + CheckURL: v.CheckURL, + Protocol: v.Protocol, + Timeout: time.Duration(v.Timeout), + Method: v.Method, + Headers: v.Headers, + Body: v.Body, + } + for _, command := range v.Commands { + check.AddCommand(command.Name, command.Message) + } + + result = append(result, check) + } + + if len(result) > 0 { + uptimeTests.CheckList = result + } } // MutualTLS contains the configuration for establishing a mutual TLS connection between Tyk and the upstream server. diff --git a/apidef/oas/upstream_test.go b/apidef/oas/upstream_test.go index 32991892fe6..d91ac05db26 100644 --- a/apidef/oas/upstream_test.go +++ b/apidef/oas/upstream_test.go @@ -147,13 +147,13 @@ func TestServiceDiscovery(t *testing.T) { assert.Equal(t, emptyServiceDiscovery, resultServiceDiscovery) } -func TestTest(t *testing.T) { - var emptyTest Test +func TestUptimeTests(t *testing.T) { + var emptyTest UptimeTests var convertedTest apidef.UptimeTests emptyTest.ExtractTo(&convertedTest) - var resultTest Test + var resultTest UptimeTests resultTest.Fill(convertedTest) assert.Equal(t, emptyTest, resultTest)