PoC: Configure features through CRD

Tiltfile

@@ -34,7 +34,7 @@ def synced_helm(path_to_chart, name, values=[]):
     """
 
     # Build the chart dependencies from the Chart.lock.
-    local('helm dep build ' + path_to_chart)
+    # local('helm dep build ' + path_to_chart)
 
     # Build the command to get the rendered kubernetes yaml.
     cmd = 'helm template'
@@ -54,7 +54,8 @@ def synced_helm(path_to_chart, name, values=[]):
 ########### Cortex Core Services
 tilt_values = os.getenv('TILT_VALUES_PATH')
 docker_build('ghcr.io/cobaltcore-dev/cortex', '.', only=[
-    'internal/', 'commands/', 'main.go', 'go.mod', 'go.sum', 'Makefile', tilt_values,
+    'internal/', 'extractor/', 'commands/',
+    'main.go', 'go.mod', 'go.sum', 'Makefile', tilt_values,
 ])
 k8s_yaml(synced_helm('./helm/cortex', name='cortex', values=[tilt_values]))
 k8s_resource('cortex-syncer', port_forwards=[
@@ -79,6 +80,10 @@ k8s_resource('cortex-kpis', port_forwards=[
     link('localhost:8004/metrics', '/metrics'),
 ], labels=['Core-Services'])
 
+########### Cortex Features
+
+k8s_yaml(synced_helm('./helm/features', name='cortex-features'))
+
 ########### Cortex Commands
 k8s_resource('cortex-cli', labels=['Commands'])
 local_resource(

extractor/api/v1/feature.go

@@ -0,0 +1,53 @@
+// Copyright 2025 SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+package v1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+// FeatureSpec defines the desired state of Feature.
+type FeatureSpec struct {
+	Foo string `json:"foo,omitempty"`
+}
+
+// FeatureStatus defines the observed state of Feature.
+type FeatureStatus struct {
+}
+
+// Feature is the Schema for the features API.
+type Feature struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   FeatureSpec   `json:"spec,omitempty"`
+	Status FeatureStatus `json:"status,omitempty"`
+}
+
+// Conform to the runtime.Object interface.
+func (in *Feature) DeepCopyObject() runtime.Object {
+	return &Feature{
+		TypeMeta:   in.TypeMeta,
+		ObjectMeta: in.ObjectMeta,
+		Spec:       in.Spec,
+		Status:     in.Status,
+	}
+}
+
+// FeatureList contains a list of Feature.
+type FeatureList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []Feature `json:"items"`
+}
+
+// Conform to the runtime.Object interface.
+func (in *FeatureList) DeepCopyObject() runtime.Object {
+	return &FeatureList{
+		TypeMeta: in.TypeMeta,
+		ListMeta: in.ListMeta,
+		Items:    in.Items,
+	}
+}

extractor/main.go

@@ -0,0 +1,85 @@
+// Copyright 2025 SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+package extractor
+
+import (
+	"crypto/tls"
+	"log/slog"
+
+	"github.com/cobaltcore-dev/cortex/internal/conf"
+	"github.com/cobaltcore-dev/cortex/internal/db"
+	"github.com/cobaltcore-dev/cortex/internal/monitoring"
+	"github.com/cobaltcore-dev/cortex/internal/mqtt"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
+
+	extractorv1 "github.com/cobaltcore-dev/cortex/extractor/api/v1"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+)
+
+func Run(registry *monitoring.Registry, config conf.FeaturesConfig, db db.DB) {
+	// Set up the kubernetes operator.
+	schemeBuilder := &scheme.Builder{GroupVersion: schema.GroupVersion{
+		Group:   "extractor.cortex.sap",
+		Version: "v1",
+	}}
+	schemeBuilder.Register(&extractorv1.Feature{}, &extractorv1.FeatureList{})
+	slog.Info("Registering scheme for feature CRD")
+	scheme, err := schemeBuilder.Build()
+	if err != nil {
+		panic("failed to build scheme: " + err.Error())
+	}
+	// If the enable-http2 flag is false (the default), http/2 should be disabled
+	// due to its vulnerabilities. More specifically, disabling http/2 will
+	// prevent from being vulnerable to the HTTP/2 Stream Cancellation and
+	// Rapid Reset CVEs. For more information see:
+	// - https://github.com/advisories/GHSA-qppj-fm5r-hxr3
+	// - https://github.com/advisories/GHSA-4374-p667-p6c8
+	webhookServer := webhook.NewServer(webhook.Options{
+		TLSOpts: []func(*tls.Config){
+			func(c *tls.Config) {
+				slog.Info("Setting up TLS for webhook server")
+				c.NextProtos = []string{"http/1.1"}
+			},
+		},
+	})
+	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+		Metrics:       metricsserver.Options{BindAddress: ":8081"}, // TODO: Conf + port + svcmonitor
+		Scheme:        scheme,
+		WebhookServer: webhookServer,
+	})
+	if err != nil {
+		panic("failed to create controller manager: " + err.Error())
+	}
+	slog.Info("Created controller manager")
+
+	// Setup the feature extractor.
+	monitor := NewPipelineMonitor(registry)
+
+	mqttClient := mqtt.NewClient(mqtt.NewMQTTMonitor(registry))
+	if err := mqttClient.Connect(); err != nil {
+		panic("failed to connect to mqtt broker: " + err.Error())
+	}
+	defer mqttClient.Disconnect()
+
+	pipeline := NewPipeline(config, db, monitor, mqttClient)
+	// Selects the extractors to run based on the config.
+	pipeline.Init(SupportedExtractors)
+	go pipeline.ExtractOnTrigger() // blocking
+
+	// Bind the reconciliation loop.
+	ctrl.NewControllerManagedBy(mgr).
+		For(&extractorv1.Feature{}).
+		Named("feature").
+		Complete(&pipeline)
+
+	slog.Info("starting manager")
+	go func() {
+		if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
+			panic("failed to start controller manager: " + err.Error())
+		}
+	}()
+}

internal/features/monitor.go → extractor/monitor.go

@@ -1,14 +1,14 @@
 // Copyright 2025 SAP SE
 // SPDX-License-Identifier: Apache-2.0
 
-package features
+package extractor
 
 import (
 	"log/slog"
 
+	"github.com/cobaltcore-dev/cortex/extractor/plugins"
 	"github.com/cobaltcore-dev/cortex/internal/conf"
 	"github.com/cobaltcore-dev/cortex/internal/db"
-	"github.com/cobaltcore-dev/cortex/internal/features/plugins"
 	"github.com/cobaltcore-dev/cortex/internal/monitoring"
 	"github.com/prometheus/client_golang/prometheus"
 )

internal/features/monitor_test.go → extractor/monitor_test.go

@@ -1,12 +1,12 @@
 // Copyright 2025 SAP SE
 // SPDX-License-Identifier: Apache-2.0
-package features
+package extractor
 
 import (
 	"strings"
 	"testing"
 
-	"github.com/cobaltcore-dev/cortex/internal/features/plugins"
+	"github.com/cobaltcore-dev/cortex/extractor/plugins"
 	"github.com/cobaltcore-dev/cortex/internal/monitoring"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/testutil"

internal/features/pipeline.go → extractor/pipeline.go

@@ -1,20 +1,24 @@
 // Copyright 2025 SAP SE
 // SPDX-License-Identifier: Apache-2.0
 
-package features
+package extractor
 
 import (
+	"context"
 	"log/slog"
 	"slices"
 	"sync"
 
+	"github.com/cobaltcore-dev/cortex/extractor/plugins"
+	"github.com/cobaltcore-dev/cortex/extractor/plugins/kvm"
+	"github.com/cobaltcore-dev/cortex/extractor/plugins/shared"
+	"github.com/cobaltcore-dev/cortex/extractor/plugins/vmware"
 	"github.com/cobaltcore-dev/cortex/internal/conf"
 	"github.com/cobaltcore-dev/cortex/internal/db"
-	"github.com/cobaltcore-dev/cortex/internal/features/plugins"
-	"github.com/cobaltcore-dev/cortex/internal/features/plugins/kvm"
-	"github.com/cobaltcore-dev/cortex/internal/features/plugins/shared"
-	"github.com/cobaltcore-dev/cortex/internal/features/plugins/vmware"
 	"github.com/cobaltcore-dev/cortex/internal/mqtt"
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	pahomqtt "github.com/eclipse/paho.mqtt.golang"
 )
@@ -36,6 +40,11 @@ var SupportedExtractors = []plugins.FeatureExtractor{
 
 // Pipeline that contains multiple feature extractors and executes them.
 type FeatureExtractorPipeline struct {
+	// Client for the kubernetes API.
+	client.Client
+	// Kubernetes scheme to use for the feature extractors.
+	Scheme *runtime.Scheme
+
 	// The dependency graph of the feature extractors, which is used to
 	// determine the execution order of the feature extractors.
 	//
@@ -203,3 +212,8 @@ func (p *FeatureExtractorPipeline) extract(order [][]plugins.FeatureExtractor) {
 		wg.Wait()
 	}
 }
+
+func (r *FeatureExtractorPipeline) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	slog.Info("reconciling feature extractor", "name", req.Name, "namespace", req.Namespace)
+	return ctrl.Result{}, nil
+}

internal/features/pipeline_test.go → extractor/pipeline_test.go

@@ -1,17 +1,17 @@
 // Copyright 2025 SAP SE
 // SPDX-License-Identifier: Apache-2.0
 
-package features
+package extractor
 
 import (
 	"errors"
 	"os"
 	"sync"
 	"testing"
 
+	"github.com/cobaltcore-dev/cortex/extractor/plugins"
 	"github.com/cobaltcore-dev/cortex/internal/conf"
 	"github.com/cobaltcore-dev/cortex/internal/db"
-	"github.com/cobaltcore-dev/cortex/internal/features/plugins"
 	"github.com/cobaltcore-dev/cortex/internal/mqtt"
 	"github.com/cobaltcore-dev/cortex/testlib/mqtt/containers"
 )

internal/features/plugins/kvm/node_exporter_host_cpu_usage.go → extractor/plugins/kvm/node_exporter_host_cpu_usage.go

@@ -6,8 +6,8 @@ package kvm
 import (
 	_ "embed"
 
+	"github.com/cobaltcore-dev/cortex/extractor/plugins"
 	"github.com/cobaltcore-dev/cortex/internal/db"
-	"github.com/cobaltcore-dev/cortex/internal/features/plugins"
 	"github.com/cobaltcore-dev/cortex/internal/sync/prometheus"
 )
 

internal/features/plugins/kvm/node_exporter_host_memory_active.go → extractor/plugins/kvm/node_exporter_host_memory_active.go

@@ -6,8 +6,8 @@ package kvm
 import (
 	_ "embed"
 
+	"github.com/cobaltcore-dev/cortex/extractor/plugins"
 	"github.com/cobaltcore-dev/cortex/internal/db"
-	"github.com/cobaltcore-dev/cortex/internal/features/plugins"
 	"github.com/cobaltcore-dev/cortex/internal/sync/prometheus"
 )
 

internal/features/plugins/shared/flavor_host_space.go → extractor/plugins/shared/flavor_host_space.go

@@ -6,8 +6,8 @@ package shared
 import (
 	_ "embed"
 
+	"github.com/cobaltcore-dev/cortex/extractor/plugins"
 	"github.com/cobaltcore-dev/cortex/internal/db"
-	"github.com/cobaltcore-dev/cortex/internal/features/plugins"
 	"github.com/cobaltcore-dev/cortex/internal/sync/openstack"
 )
 

internal/features/plugins/shared/vm_host_residency.go → extractor/plugins/shared/vm_host_residency.go

@@ -6,8 +6,8 @@ package shared
 import (
 	_ "embed"
 
+	"github.com/cobaltcore-dev/cortex/extractor/plugins"
 	"github.com/cobaltcore-dev/cortex/internal/db"
-	"github.com/cobaltcore-dev/cortex/internal/features/plugins"
 	"github.com/cobaltcore-dev/cortex/internal/sync/openstack"
 )
 

internal/features/plugins/shared/vm_life_span.go → extractor/plugins/shared/vm_life_span.go

@@ -6,8 +6,8 @@ package shared
 import (
 	_ "embed"
 
+	"github.com/cobaltcore-dev/cortex/extractor/plugins"
 	"github.com/cobaltcore-dev/cortex/internal/db"
-	"github.com/cobaltcore-dev/cortex/internal/features/plugins"
 	"github.com/cobaltcore-dev/cortex/internal/sync/openstack"
 )
 

internal/features/plugins/vmware/vrops_hostsystem_contention.go → extractor/plugins/vmware/vrops_hostsystem_contention.go

@@ -6,8 +6,8 @@ package vmware
 import (
 	_ "embed"
 
+	"github.com/cobaltcore-dev/cortex/extractor/plugins"
 	"github.com/cobaltcore-dev/cortex/internal/db"
-	"github.com/cobaltcore-dev/cortex/internal/features/plugins"
 	"github.com/cobaltcore-dev/cortex/internal/sync/prometheus"
 )
 

internal/features/plugins/vmware/vrops_hostsystem_resolver.go → extractor/plugins/vmware/vrops_hostsystem_resolver.go

@@ -6,8 +6,8 @@ package vmware
 import (
 	_ "embed"
 
+	"github.com/cobaltcore-dev/cortex/extractor/plugins"
 	"github.com/cobaltcore-dev/cortex/internal/db"
-	"github.com/cobaltcore-dev/cortex/internal/features/plugins"
 	"github.com/cobaltcore-dev/cortex/internal/sync/openstack"
 	"github.com/cobaltcore-dev/cortex/internal/sync/prometheus"
 )

internal/features/plugins/vmware/vrops_project_noisiness.go → extractor/plugins/vmware/vrops_project_noisiness.go

@@ -6,8 +6,8 @@ package vmware
 import (
 	_ "embed"
 
+	"github.com/cobaltcore-dev/cortex/extractor/plugins"
 	"github.com/cobaltcore-dev/cortex/internal/db"
-	"github.com/cobaltcore-dev/cortex/internal/features/plugins"
 	"github.com/cobaltcore-dev/cortex/internal/sync/openstack"
 	"github.com/cobaltcore-dev/cortex/internal/sync/prometheus"
 )