diff --git a/docs/pages/destinations.mdx b/docs/pages/destinations.mdx index a902f88ec..4280108aa 100644 --- a/docs/pages/destinations.mdx +++ b/docs/pages/destinations.mdx @@ -92,4 +92,6 @@ See the [building your own UI guide](/docs/guides/building-your-own-ui) for reco The destination type definitions (label, description, icon, etc) and instructions can be customized by setting the `DESTINATIONS_METADATA_PATH` environment variable to a path on disk containing the destination type definitions and instructions. Outpost will load both the default destination type definitions and any custom destination type definitions and merge them. -The metadata path is a directory containing a `providers` directory with a subdirectory for each destination type. Each destination type directory contains a `metadata.json` file and an `instructions.md` file. You can find the default destination type definitions and instructions in the [outpost-providers](https://github.com/hookdeck/outpost/tree/main/internal/destregistry/providers) folder. +> Note: Core fields (`config_fields` and `credential_fields`) cannot be overridden via custom metadata. Only non-core fields such as `label`, `description`, `icon`, and `instructions` can be customized. + +The metadata path is a directory containing a subdirectory for each destination type. Each destination type directory contains a `metadata.json` file and an `instructions.md` file. You can find the default destination type definitions and instructions in the [outpost-providers](https://github.com/hookdeck/outpost/tree/main/internal/destregistry/metadata/providers) folder. diff --git a/docs/pages/references/configuration.mdx b/docs/pages/references/configuration.mdx index 6770b757e..4a57f1c7a 100644 --- a/docs/pages/references/configuration.mdx +++ b/docs/pages/references/configuration.mdx @@ -53,7 +53,7 @@ Global configurations are provided through env variables or a YAML file. ConfigM | `DEPLOYMENT_ID` | Optional deployment identifier for multi-tenancy. Enables multiple deployments to share the same infrastructure while maintaining data isolation. | `nil` | No | | `DESTINATIONS_AWS_KINESIS_METADATA_IN_PAYLOAD` | If true, includes Outpost metadata (event ID, topic, etc.) within the Kinesis record payload. | `true` | No | | `DESTINATIONS_INCLUDE_MILLISECOND_TIMESTAMP` | If true, includes a 'timestamp-ms' field with millisecond precision in destination metadata. Useful for load testing and debugging. | `false` | No | -| `DESTINATIONS_METADATA_PATH` | Path to the directory containing custom destination type definitions. This can be overridden by the root-level 'destination_metadata_path' if also set. | `config/outpost/destinations` | No | +| `DESTINATIONS_METADATA_PATH` | Path to the directory containing custom destination type definitions. | `config/outpost/destinations` | No | | `DESTINATIONS_WEBHOOK_DISABLE_DEFAULT_EVENT_ID_HEADER` | If true, disables adding the default 'X-Outpost-Event-Id' header to webhook requests. Only applies to 'default' mode. | `false` | No | | `DESTINATIONS_WEBHOOK_DISABLE_DEFAULT_SIGNATURE_HEADER` | If true, disables adding the default 'X-Outpost-Signature' header to webhook requests. Only applies to 'default' mode. | `false` | No | | `DESTINATIONS_WEBHOOK_DISABLE_DEFAULT_TIMESTAMP_HEADER` | If true, disables adding the default 'X-Outpost-Timestamp' header to webhook requests. Only applies to 'default' mode. | `false` | No | @@ -65,7 +65,6 @@ Global configurations are provided through env variables or a YAML file. ConfigM | `DESTINATIONS_WEBHOOK_SIGNATURE_CONTENT_TEMPLATE` | Go template for constructing the content to be signed for webhook requests. Only applies to 'default' mode. | `{{.Body}}` | No | | `DESTINATIONS_WEBHOOK_SIGNATURE_ENCODING` | Encoding for the signature (e.g., 'hex', 'base64'). Only applies to 'default' mode. | `hex` | No | | `DESTINATIONS_WEBHOOK_SIGNATURE_HEADER_TEMPLATE` | Go template for the value of the signature header. Only applies to 'default' mode. | `v0={{.Signatures \| join ","}}` | No | -| `DESTINATION_METADATA_PATH` | Path to the directory containing custom destination type definitions. Overrides 'destinations.metadata_path' if set. | `nil` | No | | `DISABLE_TELEMETRY` | Global flag to disable all telemetry (anonymous usage statistics to Hookdeck and error reporting to Sentry). If true, overrides 'telemetry.disabled'. | `false` | No | | `GCP_PUBSUB_DELIVERY_SUBSCRIPTION` | Name of the GCP Pub/Sub subscription for delivery events. | `outpost-delivery-sub` | No | | `GCP_PUBSUB_DELIVERY_TOPIC` | Name of the GCP Pub/Sub topic for delivery events. | `outpost-delivery` | No | @@ -212,9 +211,6 @@ delivery_timeout_seconds: 5 # Optional deployment identifier for multi-tenancy. Enables multiple deployments to share the same infrastructure while maintaining data isolation. deployment_id: "" -# Path to the directory containing custom destination type definitions. Overrides 'destinations.metadata_path' if set. -destination_metadata_path: "" - destinations: # Configuration specific to AWS Kinesis destinations. aws_kinesis: @@ -225,7 +221,7 @@ destinations: # If true, includes a 'timestamp-ms' field with millisecond precision in destination metadata. Useful for load testing and debugging. include_millisecond_timestamp: false - # Path to the directory containing custom destination type definitions. This can be overridden by the root-level 'destination_metadata_path' if also set. + # Path to the directory containing custom destination type definitions. metadata_path: "config/outpost/destinations" # Configuration specific to webhook destinations. diff --git a/internal/config/config.go b/internal/config/config.go index edfa8658e..8a206100e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -90,9 +90,6 @@ type Config struct { PublishIdempotencyKeyTTL int `yaml:"publish_idempotency_key_ttl" env:"PUBLISH_IDEMPOTENCY_KEY_TTL" desc:"Time-to-live in seconds for publish queue idempotency keys. Controls how long processed events are remembered to prevent duplicate processing. Default: 3600 (1 hour)." required:"N"` DeliveryIdempotencyKeyTTL int `yaml:"delivery_idempotency_key_ttl" env:"DELIVERY_IDEMPOTENCY_KEY_TTL" desc:"Time-to-live in seconds for delivery queue idempotency keys. Controls how long processed deliveries are remembered to prevent duplicate delivery attempts. Default: 3600 (1 hour)." required:"N"` - // Destination Registry - DestinationMetadataPath string `yaml:"destination_metadata_path" env:"DESTINATION_METADATA_PATH" desc:"Path to the directory containing custom destination type definitions. Overrides 'destinations.metadata_path' if set." required:"N"` - // Log batcher configuration LogBatchThresholdSeconds int `yaml:"log_batch_threshold_seconds" env:"LOG_BATCH_THRESHOLD_SECONDS" desc:"Maximum time in seconds to buffer logs before flushing them to storage, if batch size is not reached." required:"N"` LogBatchSize int `yaml:"log_batch_size" env:"LOG_BATCH_SIZE" desc:"Maximum number of log entries to batch together before writing to storage." required:"N"` diff --git a/internal/config/destinations.go b/internal/config/destinations.go index 1d9aa9260..4673d48f7 100644 --- a/internal/config/destinations.go +++ b/internal/config/destinations.go @@ -9,7 +9,7 @@ import ( // DestinationsConfig is the main configuration for all destination types type DestinationsConfig struct { - MetadataPath string `yaml:"metadata_path" env:"DESTINATIONS_METADATA_PATH" desc:"Path to the directory containing custom destination type definitions. This can be overridden by the root-level 'destination_metadata_path' if also set." required:"N"` + MetadataPath string `yaml:"metadata_path" env:"DESTINATIONS_METADATA_PATH" desc:"Path to the directory containing custom destination type definitions." required:"N"` IncludeMillisecondTimestamp bool `yaml:"include_millisecond_timestamp" env:"DESTINATIONS_INCLUDE_MILLISECOND_TIMESTAMP" desc:"If true, includes a 'timestamp-ms' field with millisecond precision in destination metadata. Useful for load testing and debugging." required:"N"` Webhook DestinationWebhookConfig `yaml:"webhook" desc:"Configuration specific to webhook destinations."` AWSKinesis DestinationAWSKinesisConfig `yaml:"aws_kinesis" desc:"Configuration specific to AWS Kinesis destinations."` diff --git a/internal/destregistry/metadata/providers/aws_kinesis/metadata.json b/internal/destregistry/metadata/providers/aws_kinesis/metadata.json index dcf40a87d..87d2756c4 100644 --- a/internal/destregistry/metadata/providers/aws_kinesis/metadata.json +++ b/internal/destregistry/metadata/providers/aws_kinesis/metadata.json @@ -26,7 +26,7 @@ "label": "Endpoint", "description": "Custom endpoint URL for AWS Kinesis (optional, for testing or VPC endpoints)", "required": false, - "pattern": "^https?:\\/\\/[\\w\\-]+(?:\\.[\\w\\-]+)*(?::\\d{1,5})?(?:\\/[\\w\\-\\/\\.~:%?#\\[\\]@!$&'\\(\\)*+,;=]*)?$" + "pattern": "^https?:\\/\\/[^\\s]+$" }, { "key": "partition_key_template", diff --git a/internal/destregistry/metadata/providers/aws_sqs/metadata.json b/internal/destregistry/metadata/providers/aws_sqs/metadata.json index 99f130fc4..be628d22f 100644 --- a/internal/destregistry/metadata/providers/aws_sqs/metadata.json +++ b/internal/destregistry/metadata/providers/aws_sqs/metadata.json @@ -7,7 +7,7 @@ "label": "Queue URL", "description": "The URL of your AWS SQS queue", "required": true, - "pattern": "^https?:\\/\\/[\\w\\-]+(?:\\.[\\w\\-]+)*(?::\\d{1,5})?(?:\\/[\\w\\-\\/\\.~:%?#\\[\\]@!$&'\\(\\)*+,;=]*)?$" + "pattern": "^https?:\\/\\/[^\\s]+$" } ], "credential_fields": [ diff --git a/internal/destregistry/metadata/providers/rabbitmq/metadata.json b/internal/destregistry/metadata/providers/rabbitmq/metadata.json index d499f5129..264422fc6 100644 --- a/internal/destregistry/metadata/providers/rabbitmq/metadata.json +++ b/internal/destregistry/metadata/providers/rabbitmq/metadata.json @@ -7,7 +7,7 @@ "label": "Server URL", "description": "The RabbitMQ server URL (e.g., myrabbitmq.com:5672)", "required": true, - "pattern": "^(?:localhost|127\\.0\\.0\\.1|\\[::1\\]|(?:[\\w\\-]+\\.)+[a-z]{2,})(?::\\d{1,5})?$" + "pattern": "^[^\\s]+$" }, { "key": "exchange", diff --git a/internal/destregistry/metadata/providers/webhook/metadata.json b/internal/destregistry/metadata/providers/webhook/metadata.json index 8227d8883..a3347f991 100644 --- a/internal/destregistry/metadata/providers/webhook/metadata.json +++ b/internal/destregistry/metadata/providers/webhook/metadata.json @@ -7,7 +7,7 @@ "label": "Webhook URL", "description": "The URL to send webhook events to via HTTP POST", "required": true, - "pattern": "^https?:\\/\\/[\\w\\-]+(?:\\.[\\w\\-]+)*(?::\\d{1,5})?(?:\\/[\\w\\-\\/\\.~:%?#\\[\\]@!$&'\\(\\)*+,;=]*)?$" + "pattern": "^https?:\\/\\/[^\\s]+$" }, { "key": "custom_headers", diff --git a/internal/destregistry/metadata/providers/webhook_standard/metadata.json b/internal/destregistry/metadata/providers/webhook_standard/metadata.json index 09b501452..82023988f 100644 --- a/internal/destregistry/metadata/providers/webhook_standard/metadata.json +++ b/internal/destregistry/metadata/providers/webhook_standard/metadata.json @@ -7,7 +7,7 @@ "label": "Webhook URL", "description": "The URL to send webhook events to via HTTP POST (Standard Webhooks compliant)", "required": true, - "pattern": "^https?:\\/\\/[\\w\\-]+(?:\\.[\\w\\-]+)*(?::\\d{1,5})?(?:\\/[\\w\\-\\/\\.~:%?#\\[\\]@!$&'\\(\\)*+,;=]*)?$" + "pattern": "^https?:\\/\\/[^\\s]+$" }, { "key": "custom_headers", diff --git a/internal/destregistry/providers/destrabbitmq/destrabbitmq_validate_test.go b/internal/destregistry/providers/destrabbitmq/destrabbitmq_validate_test.go index e28695bd3..ba2c0ab6e 100644 --- a/internal/destregistry/providers/destrabbitmq/destrabbitmq_validate_test.go +++ b/internal/destregistry/providers/destrabbitmq/destrabbitmq_validate_test.go @@ -75,19 +75,53 @@ func TestRabbitMQDestination_Validate(t *testing.T) { assert.Equal(t, "required", validationErr.Errors[0].Type) }) - t.Run("should validate malformed server_url", func(t *testing.T) { + t.Run("should accept valid server URLs", func(t *testing.T) { t.Parallel() - dest := validDestination - dest.Config = map[string]string{ - "server_url": "not-a-valid-url", - "exchange": "test-exchange", + validURLs := []string{ + "localhost:5672", + "127.0.0.1:5672", + "[::1]:5672", + // Docker service names + "rabbitmq:5672", + "my-rabbitmq:5672", + // FQDN + "rabbitmq.example.com:5672", + "mq.internal.local:5672", + // Without port + "rabbitmq.example.com", + "rabbitmq", + // IP addresses + "192.168.1.100:5672", + "10.0.0.1:5672", + } + for _, url := range validURLs { + t.Run(url, func(t *testing.T) { + t.Parallel() + dest := validDestination + dest.Config = map[string]string{"server_url": url, "exchange": "test-exchange"} + dest.Credentials = maps.Clone(validDestination.Credentials) + assert.NoError(t, rabbitmqDestination.Validate(context.Background(), &dest)) + }) + } + }) + + t.Run("should reject invalid server URLs", func(t *testing.T) { + t.Parallel() + invalidURLs := []string{ + "", + "host with spaces:5672", + "host name:5672", + } + for _, url := range invalidURLs { + t.Run(url, func(t *testing.T) { + t.Parallel() + dest := validDestination + dest.Config = map[string]string{"server_url": url, "exchange": "test-exchange"} + dest.Credentials = maps.Clone(validDestination.Credentials) + err := rabbitmqDestination.Validate(context.Background(), &dest) + assert.Error(t, err) + }) } - dest.Credentials = maps.Clone(validDestination.Credentials) - err := rabbitmqDestination.Validate(context.Background(), &dest) - var validationErr *destregistry.ErrDestinationValidation - assert.ErrorAs(t, err, &validationErr) - assert.Equal(t, "config.server_url", validationErr.Errors[0].Field) - assert.Equal(t, "pattern", validationErr.Errors[0].Type) }) t.Run("should validate valid destination without exchange", func(t *testing.T) { diff --git a/internal/destregistry/providers/destwebhook/destwebhook_validate_test.go b/internal/destregistry/providers/destwebhook/destwebhook_validate_test.go index 670186fe1..49aacb6a6 100644 --- a/internal/destregistry/providers/destwebhook/destwebhook_validate_test.go +++ b/internal/destregistry/providers/destwebhook/destwebhook_validate_test.go @@ -76,17 +76,28 @@ func TestWebhookDestination_Validate(t *testing.T) { t.Run("should accept valid URLs", func(t *testing.T) { t.Parallel() validURLs := []string{ + // Standard URLs "https://example.com", + "http://example.com", "https://example.com/path", "https://example.com:8080/path", "https://example.com/path?query=value", "https://example.com/path#fragment", "https://sub.example.com/path", "http://localhost:3000/webhook", + // Basic Auth URLs + "https://user:pass@example.com", + "https://user:pass@example.com/path", + "https://user:pass@example.com:8080/path", + "https://token@example.com/webhook", + "https://sam:123444@example.com/api/message", // Percent-encoded URLs (Azure Logic Apps, etc.) "https://example.com/path?param=%2Fencoded%2Fslash", "https://example.com/path%2Fwith%2Fencoded", "https://logic.azure.com/workflows/abc123/triggers/manual?api-version=2016&sp=%2Ftriggers%2Fmanual%2Frun", + // IP addresses + "http://192.168.1.1:8080/webhook", + "http://127.0.0.1/webhook", } for _, url := range validURLs { t.Run(url, func(t *testing.T) { @@ -106,6 +117,7 @@ func TestWebhookDestination_Validate(t *testing.T) { "://missing-scheme.com", "https://", "", + "example.com", } for _, url := range invalidURLs { t.Run(url, func(t *testing.T) { diff --git a/internal/destregistry/providers/destwebhookstandard/destwebhookstandard_validate_test.go b/internal/destregistry/providers/destwebhookstandard/destwebhookstandard_validate_test.go index bf210c69d..499ba8109 100644 --- a/internal/destregistry/providers/destwebhookstandard/destwebhookstandard_validate_test.go +++ b/internal/destregistry/providers/destwebhookstandard/destwebhookstandard_validate_test.go @@ -73,6 +73,63 @@ func TestStandardWebhookDestination_Validate(t *testing.T) { assert.Equal(t, "pattern", validationErr.Errors[0].Type) }) + t.Run("should accept valid URLs", func(t *testing.T) { + t.Parallel() + validURLs := []string{ + // Standard URLs + "https://example.com", + "http://example.com", + "https://example.com/path", + "https://example.com:8080/path", + "https://example.com/path?query=value", + "https://example.com/path#fragment", + "https://sub.example.com/path", + "http://localhost:3000/webhook", + // Basic Auth URLs + "https://user:pass@example.com", + "https://user:pass@example.com/path", + "https://user:pass@example.com:8080/path", + "https://token@example.com/webhook", + "https://sam:123444@example.com/api/message", + // Percent-encoded URLs (Azure Logic Apps, etc.) + "https://example.com/path?param=%2Fencoded%2Fslash", + "https://example.com/path%2Fwith%2Fencoded", + "https://logic.azure.com/workflows/abc123/triggers/manual?api-version=2016&sp=%2Ftriggers%2Fmanual%2Frun", + // IP addresses + "http://192.168.1.1:8080/webhook", + "http://127.0.0.1/webhook", + } + for _, url := range validURLs { + t.Run(url, func(t *testing.T) { + t.Parallel() + dest := validDestination + dest.Config = map[string]string{"url": url} + assert.NoError(t, provider.Validate(context.Background(), &dest)) + }) + } + }) + + t.Run("should reject invalid URLs", func(t *testing.T) { + t.Parallel() + invalidURLs := []string{ + "not-a-url", + "ftp://example.com", + "://missing-scheme.com", + "https://", + "", + "example.com", + } + for _, url := range invalidURLs { + t.Run(url, func(t *testing.T) { + t.Parallel() + dest := validDestination + dest.Config = map[string]string{"url": url} + err := provider.Validate(context.Background(), &dest) + assert.Error(t, err) + }) + } + }) + t.Run("should validate secret without whsec prefix", func(t *testing.T) { t.Parallel() invalidDestination := validDestination