[Ingestion Controller ] Add compaction, rules, dynamic configurations…

…, nativeSpec (#197)

* Add compaction, rules, dynamic configurations, nativeSpec

* update strings in EmitEventGeneric() calls within dynamic_config.go to use constants, set MinIO version to avoid unexpected breaking upgrades

Makefile

@@ -11,6 +11,9 @@ NAMESPACE_DRUID_OPERATOR ?= "druid-operator-system"
 NAMESPACE_ZK_OPERATOR ?= "zk-operator"
 # NAMESPACE for zk operator e2e
 NAMESPACE_MINIO_OPERATOR ?= "minio-operator"
+# MinIO version to install. Specific version is set to avoid breaking changes
+# using latest version. This version is tested with the operator.
+MINIO_VERSION ?= "6.0.4"
 # NAMESPACE for druid app e2e
 NAMESPACE_DRUID ?= "druid"
 
@@ -116,9 +119,11 @@ helm-minio-install: ## Helm deploy minio operator and minio
 	--namespace ${NAMESPACE_MINIO_OPERATOR} \
 	--create-namespace \
 	 ${NAMESPACE_MINIO_OPERATOR} minio/operator \
+	--version ${MINIO_VERSION} \
 	-f e2e/configs/minio-operator-override.yaml
 	helm upgrade --install \
 	--namespace ${NAMESPACE_DRUID} \
+	--version ${MINIO_VERSION} \
 	--create-namespace \
   	${NAMESPACE_DRUID}-minio minio/tenant \
 	-f e2e/configs/minio-tenant-override.yaml

apis/druid/v1alpha1/druid_types.go

@@ -3,12 +3,14 @@ package v1alpha1
 import (
 	"encoding/json"
 
+	druidapi "github.com/datainfrahq/druid-operator/pkg/druidapi"
 	appsv1 "k8s.io/api/apps/v1"
 	autoscalev2 "k8s.io/api/autoscaling/v2"
 	v1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
 	policyv1 "k8s.io/api/policy/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 )
 
 // druid-operator deploys a druid cluster from given spec below, based on the spec it would create following
@@ -281,6 +283,13 @@ type DruidSpec struct {
 	// CoreSite Contents of `core-site.xml`.
 	// +optional
 	CoreSite string `json:"core-site.xml,omitempty"`
+
+	// Dynamic Configurations for Druid. Applied through the dynamic configuration API.
+	// +optional
+	DynamicConfig runtime.RawExtension `json:"dynamicConfig,omitempty"`
+
+	// +optional
+	Auth druidapi.Auth `json:"auth,omitempty"`
 }
 
 // DruidNodeSpec Specification of `Druid` Node type and its configurations.
@@ -481,6 +490,10 @@ type DruidNodeSpec struct {
 	// ServiceAccountName Kubernetes native `serviceAccountName` specification.
 	// +optional
 	ServiceAccountName string `json:"serviceAccountName,omitempty"`
+
+	// Dynamic Configurations for Druid. Applied through the dynamic configuration API.
+	// +optional
+	DynamicConfig runtime.RawExtension `json:"dynamicConfig,omitempty"`
 }
 
 // ZookeeperSpec IGNORED (Future API): In order to make Druid dependency setup extensible from within Druid operator.

apis/druid/v1alpha1/druidingestion_types.go

@@ -17,8 +17,10 @@ limitations under the License.
 package v1alpha1
 
 import (
+	druidapi "github.com/datainfrahq/druid-operator/pkg/druidapi"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 )
 
 type DruidIngestionMethod string
@@ -39,37 +41,39 @@ type DruidIngestionSpec struct {
 	// +required
 	Ingestion IngestionSpec `json:"ingestion"`
 	// +optional
-	Auth Auth `json:"auth"`
+	Auth druidapi.Auth `json:"auth"`
 }
 
 type IngestionSpec struct {
 	// +required
 	Type DruidIngestionMethod `json:"type"`
-	// +required
+	// +optional
+	// Spec should be passed in as a JSON string.
+	// Note: This field is planned for deprecation in favor of nativeSpec.
 	Spec string `json:"spec,omitempty"`
+	// +optional
+	// nativeSpec allows the ingestion specification to be defined in a native Kubernetes format.
+	// This is particularly useful for environment-specific configurations and will eventually
+	// replace the JSON-based Spec field.
+	// Note: Spec will be ignored if nativeSpec is provided.
+	NativeSpec runtime.RawExtension `json:"nativeSpec,omitempty"`
+	// +optional
+	Compaction runtime.RawExtension `json:"compaction,omitempty"`
+	// +optional
+	Rules []runtime.RawExtension `json:"rules,omitempty"`
 }
 
 type DruidIngestionStatus struct {
-	TaskId               string             `json:"taskId"`
-	Type                 string             `json:"type,omitempty"`
-	Status               v1.ConditionStatus `json:"status,omitempty"`
-	Reason               string             `json:"reason,omitempty"`
-	Message              string             `json:"message,omitempty"`
-	LastUpdateTime       metav1.Time        `json:"lastUpdateTime,omitempty"`
-	CurrentIngestionSpec string             `json:"currentIngestionSpec.json"`
-}
-
-type AuthType string
-
-const (
-	BasicAuth AuthType = "basic-auth"
-)
-
-type Auth struct {
-	// +required
-	Type AuthType `json:"type"`
-	// +required
-	SecretRef v1.SecretReference `json:"secretRef"`
+	TaskId         string             `json:"taskId"`
+	Type           string             `json:"type,omitempty"`
+	Status         v1.ConditionStatus `json:"status,omitempty"`
+	Reason         string             `json:"reason,omitempty"`
+	Message        string             `json:"message,omitempty"`
+	LastUpdateTime metav1.Time        `json:"lastUpdateTime,omitempty"`
+	// CurrentIngestionSpec is a string instead of RawExtension to maintain compatibility with existing
+	// IngestionSpecs that are stored as JSON strings.
+	CurrentIngestionSpec string                 `json:"currentIngestionSpec.json"`
+	CurrentRules         []runtime.RawExtension `json:"rules,omitempty"`
 }
 
 // +kubebuilder:object:root=true

chart/crds/druid.apache.org_druidingestions.yaml

@@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.2
-  creationTimestamp: null
+    controller-gen.kubebuilder.io/version: v0.14.0
   name: druidingestions.druid.apache.org
 spec:
   group: druid.apache.org
@@ -28,14 +27,19 @@ spec:
         description: Ingestion is the Schema for the Ingestion API
         properties:
           apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
-              of an object. Servers should convert recognized schemas to the latest
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
             type: string
           kind:
-            description: 'Kind is a string value representing the REST resource this
-              object represents. Servers may infer this from the endpoint the client
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
             type: string
           metadata:
             type: object
@@ -44,8 +48,9 @@ spec:
               auth:
                 properties:
                   secretRef:
-                    description: SecretReference represents a Secret Reference. It
-                      has enough information to retrieve secret in any namespace
+                    description: |-
+                      SecretReference represents a Secret Reference. It has enough information to retrieve secret
+                      in any namespace
                     properties:
                       name:
                         description: name is unique within a namespace to reference
@@ -67,7 +72,26 @@ spec:
                 type: string
               ingestion:
                 properties:
+                  compaction:
+                    type: object
+                    x-kubernetes-preserve-unknown-fields: true
+                  nativeSpec:
+                    description: |-
+                      nativeSpec allows the ingestion specification to be defined in a native Kubernetes format.
+                      This is particularly useful for environment-specific configurations and will eventually
+                      replace the JSON-based Spec field.
+                      Note: Spec will be ignored if nativeSpec is provided.
+                    type: object
+                    x-kubernetes-preserve-unknown-fields: true
+                  rules:
+                    items:
+                      type: object
+                      x-kubernetes-preserve-unknown-fields: true
+                    type: array
                   spec:
+                    description: |-
+                      Spec should be passed in as a JSON string.
+                      Note: This field is planned for deprecation in favor of nativeSpec.
                     type: string
                   type:
                     type: string
@@ -83,6 +107,9 @@ spec:
           status:
             properties:
               currentIngestionSpec.json:
+                description: |-
+                  CurrentIngestionSpec is a string instead of RawExtension to maintain compatibility with existing
+                  IngestionSpecs that are stored as JSON strings.
                 type: string
               lastUpdateTime:
                 format: date-time
@@ -91,6 +118,11 @@ spec:
                 type: string
               reason:
                 type: string
+              rules:
+                items:
+                  type: object
+                  x-kubernetes-preserve-unknown-fields: true
+                type: array
               status:
                 type: string
               taskId: