Ease config of multiple prometheuses for same metric type (#49)

Author: PhilippMatthes

cortex.secrets.example.yaml

@@ -26,9 +26,15 @@ conf:
     prometheus:
       hosts:
         # See: https://github.com/sapcc/vrops-exporter
-        - name: vrops_prometheus
+        - name: vmware_prometheus
           url: https://path-to-your-vrops-prometheus
           sso: *sharedSSOCert
+          provides: [vrops_vm_metric, vrops_host_metric]
+        # See: https://github.com/prometheus/node_exporter
+        - name: kvm_prometheus
+          url: https://path-to-your-node-exporter
+          sso: *sharedSSOCert
+          provides: [node_exporter_metric]
     # Override the endpoints and credentials to your OpenStack.
     openstack:
       keystone:

helm/cortex/Chart.yaml

@@ -18,7 +18,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.3.0
+version: 0.4.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/cortex/values.yaml

@@ -135,12 +135,14 @@ conf:
     prometheus:
       # Prometheus hosts to consider when syncing the metrics.
       hosts:
-        - name: vrops_prometheus
+        - name: vmware_prometheus
           url: https://path-to-vrops-prometheus
+          provides: [vrops_vm_metric, vrops_host_metric]
           # Optional, only required if Prometheus is configured with SSO.
           sso: *sharedSSOCert
-        - name: node_exporter_prometheus
+        - name: kvm_prometheus
           url: https://path-to-node-exporter
+          provides: [node_exporter_metric]
           # Optional, only required if Prometheus is configured with SSO.
           sso: *sharedSSOCert
       # Prometheus metrics to sync into the database.
@@ -152,12 +154,10 @@ conf:
           query: |
             vrops_virtualmachine_cpu_demand_ratio{}
           type: vrops_vm_metric
-          prometheusNames: [vrops_prometheus]
         - alias: vrops_hostsystem_cpu_contention_percentage
           query: |
             vrops_hostsystem_cpu_contention_percentage{}
           type: vrops_host_metric
-          prometheusNames: [vrops_prometheus]
         # Node exporter metrics for KVM hosts.
         - alias: node_exporter_cpu_usage_pct
           query: |
@@ -166,7 +166,6 @@ conf:
               count by (node) (rate(node_cpu_seconds_total{mode="idle"}[1m]))
             )) * 100
           type: node_exporter_metric
-          prometheusNames: [node_exporter_prometheus]
 
     openstack:
       keystone:

internal/conf/validation.go

@@ -134,5 +134,21 @@ func (c *config) Validate() error {
 	if !slices.Contains(validAvailabilities, c.SyncConfig.OpenStack.Placement.Availability) {
 		return fmt.Errorf("invalid placement availability %s", c.SyncConfig.OpenStack.Placement.Availability)
 	}
+	// Check that all confed metric types have a host to sync from.
+	confedMetricTypes := make(map[string]bool)
+	for _, metric := range c.SyncConfig.Prometheus.Metrics {
+		confedMetricTypes[metric.Type] = true
+	}
+	providedMetricTypes := make(map[string]bool)
+	for _, host := range c.SyncConfig.Prometheus.Hosts {
+		for _, metricType := range host.ProvidedMetricTypes {
+			providedMetricTypes[metricType] = true
+		}
+	}
+	for metricType := range confedMetricTypes {
+		if !providedMetricTypes[metricType] {
+			return fmt.Errorf("no host provided for metric type %s", metricType)
+		}
+	}
 	return nil
 }

internal/conf/validation_test.go

@@ -9,6 +9,10 @@ func TestValidConf(t *testing.T) {
 	content := `
 sync:
   prometheus:
+    hosts:
+      - name: prometheus
+        url: http://prometheus:9090
+        provides: [metric_type_1]
     metrics:
       - alias: metric_1
         query: metric_1
@@ -99,3 +103,18 @@ sync:
 		t.Fatalf("expected error, got nil")
 	}
 }
+
+func TestInvalidConf_MissingHost(t *testing.T) {
+	content := `
+sync:
+  prometheus:
+    metrics:
+      - alias: metric_1
+        query: metric_1
+        type: metric_type_1
+`
+	conf := newConfigFromBytes([]byte(content))
+	if err := conf.Validate(); err == nil {
+		t.Fatalf("expected error, got nil")
+	}
+}

internal/conf/yaml.go

@@ -51,11 +51,6 @@ type SyncPrometheusMetricConfig struct {
 	TimeRangeSeconds  *int `yaml:"timeRangeSeconds,omitempty"`
 	IntervalSeconds   *int `yaml:"intervalSeconds,omitempty"`
 	ResolutionSeconds *int `yaml:"resolutionSeconds,omitempty"`
-
-	// One or multiple prometheus names to fetch the metric from.
-	// If multiple names are given, the metric will be fetched from
-	// all of them.
-	PrometheusNames []string `yaml:"prometheusNames"`
 }
 
 // Configuration for a single prometheus host.
@@ -66,6 +61,8 @@ type SyncPrometheusHostConfig struct {
 	URL string `yaml:"url"`
 	// The SSO configuration for this host.
 	SSO SSOConfig `yaml:"sso,omitempty"`
+	// The types of metrics this host provides.
+	ProvidedMetricTypes []string `yaml:"provides"`
 }
 
 // Configuration for the sync/prometheus module containing a list of metrics.

internal/sync/prometheus/sync.go

@@ -62,22 +62,19 @@ type CombinedSyncer struct {
 func NewCombinedSyncer(config conf.SyncPrometheusConfig, db db.DB, monitor sync.Monitor) sync.Datasource {
 	slog.Info("loading syncers", "metrics", config.Metrics)
 	syncers := []sync.Datasource{}
-	hostConfByName := make(map[string]conf.SyncPrometheusHostConfig)
-	for _, hostConf := range config.Hosts {
-		hostConfByName[hostConf.Name] = hostConf
-	}
 	for _, metricConfig := range config.Metrics {
 		syncerFunc, ok := supportedSyncers[metricConfig.Type]
 		if !ok {
 			panic("unsupported metric type: " + metricConfig.Type)
 		}
-		// Sync the metric from one or multiple prometheus hosts.
-		for _, prometheusName := range metricConfig.PrometheusNames {
-			hostConf, ok := hostConfByName[prometheusName]
-			if !ok {
-				panic("unknown metric host: " + prometheusName)
+		// Get the prometheuses to sync this metric from.
+		for _, hostConf := range config.Hosts {
+			for _, providedMetricType := range hostConf.ProvidedMetricTypes {
+				if providedMetricType == metricConfig.Type {
+					slog.Info("adding syncer", "metricType", metricConfig.Type, "host", hostConf.Name)
+					syncers = append(syncers, syncerFunc(db, hostConf, metricConfig, monitor))
+				}
 			}
-			syncers = append(syncers, syncerFunc(db, hostConf, metricConfig, monitor))
 		}
 	}
 	return CombinedSyncer{