NETOBSERV-2477: network predefined metrics + add missing ipsec

api/flowcollector/v1beta2/flowcollector_types.go

@@ -564,7 +564,7 @@ type MetricsServerConfig struct {
 }
 
 // Metric name. More information in https://github.com/netobserv/network-observability-operator/blob/main/docs/Metrics.md.
-// +kubebuilder:validation:Enum:="namespace_egress_bytes_total";"namespace_egress_packets_total";"namespace_ingress_bytes_total";"namespace_ingress_packets_total";"namespace_flows_total";"node_egress_bytes_total";"node_egress_packets_total";"node_ingress_bytes_total";"node_ingress_packets_total";"node_flows_total";"workload_egress_bytes_total";"workload_egress_packets_total";"workload_ingress_bytes_total";"workload_ingress_packets_total";"workload_flows_total";"namespace_drop_bytes_total";"namespace_drop_packets_total";"node_drop_bytes_total";"node_drop_packets_total";"workload_drop_bytes_total";"workload_drop_packets_total";"namespace_rtt_seconds";"node_rtt_seconds";"workload_rtt_seconds";"namespace_dns_latency_seconds";"node_dns_latency_seconds";"workload_dns_latency_seconds";"node_network_policy_events_total";"namespace_network_policy_events_total";"workload_network_policy_events_total";"node_ipsec_flows_total";"node_to_node_ingress_flows_total"
+// +kubebuilder:validation:Enum:="namespace_egress_bytes_total";"namespace_egress_packets_total";"namespace_ingress_bytes_total";"namespace_ingress_packets_total";"namespace_flows_total";"node_egress_bytes_total";"node_egress_packets_total";"node_ingress_bytes_total";"node_ingress_packets_total";"node_flows_total";"workload_egress_bytes_total";"workload_egress_packets_total";"workload_ingress_bytes_total";"workload_ingress_packets_total";"workload_flows_total";"namespace_drop_bytes_total";"namespace_drop_packets_total";"node_drop_bytes_total";"node_drop_packets_total";"workload_drop_bytes_total";"workload_drop_packets_total";"namespace_rtt_seconds";"node_rtt_seconds";"workload_rtt_seconds";"namespace_dns_latency_seconds";"node_dns_latency_seconds";"workload_dns_latency_seconds";"node_network_policy_events_total";"namespace_network_policy_events_total";"workload_network_policy_events_total";"node_ipsec_flows_total";"namespace_ipsec_flows_total";"workload_ipsec_flows_total";"node_to_node_ingress_flows_total"
 type FLPMetric string
 
 // `FLPMetrics` define the desired FLP configuration regarding metrics

bundle/manifests/flows.netobserv.io_flowcollectors.yaml

@@ -5994,6 +5994,8 @@ spec:
                           - namespace_network_policy_events_total
                           - workload_network_policy_events_total
                           - node_ipsec_flows_total
+                          - namespace_ipsec_flows_total
+                          - workload_ipsec_flows_total
                           - node_to_node_ingress_flows_total
                           type: string
                         type: array

config/crd/bases/flows.netobserv.io_flowcollectors.yaml

@@ -5546,6 +5546,8 @@ spec:
                               - namespace_network_policy_events_total
                               - workload_network_policy_events_total
                               - node_ipsec_flows_total
+                              - namespace_ipsec_flows_total
+                              - workload_ipsec_flows_total
                               - node_to_node_ingress_flows_total
                             type: string
                           type: array

helm/crds/flows.netobserv.io_flowcollectors.yaml

@@ -5550,6 +5550,8 @@ spec:
                               - namespace_network_policy_events_total
                               - workload_network_policy_events_total
                               - node_ipsec_flows_total
+                              - namespace_ipsec_flows_total
+                              - workload_ipsec_flows_total
                               - node_to_node_ingress_flows_total
                             type: string
                           type: array

internal/pkg/dashboards/dashboard_test.go

@@ -105,6 +105,7 @@ func TestCreateFlowMetricsDashboard_DefaultList(t *testing.T) {
 		"TCP latencies",
 		"Byte and packet drops",
 		"DNS",
+		"IPsec",
 	}, d.Titles())
 
 	topRow := d.FindRow("")
@@ -115,6 +116,7 @@ func TestCreateFlowMetricsDashboard_DefaultList(t *testing.T) {
 		"Drops",
 		"DNS latency, p99",
 		"DNS error rate",
+		"IPsec encrypted traffic",
 	}, topRow.Titles())
 
 	trafficRow := d.FindRow("Traffic rates per node")

internal/pkg/helper/otel/otel-config.json

@@ -10,6 +10,7 @@
   "SrcK8S_HostIP": "source.k8s.host.address",
   "SrcK8S_HostName": "source.k8s.host.name",
   "SrcK8S_Zone": "source.zone",
+  "SrcK8S_NetworkName": "source.network.name",
   "SrcSubnetLabel": "source.subnet.label",
   "DstAddr": "destination.address",
   "DstMac": "destination.mac",
@@ -22,6 +23,7 @@
   "DstK8S_HostIP": "destination.k8s.host.address",
   "DstK8S_HostName": "destination.k8s.host.name",
   "DstK8S_Zone": "destination.zone",
+  "DstK8S_NetworkName": "destination.network.name",
   "DstSubnetLabel": "destination.subnet.label",
   "Bytes": "bytes",
   "Packets": "packets",
@@ -30,6 +32,7 @@
   "TimeFlowRttNs": "tcp.rtt",
   "Interfaces": "interface.names",
   "IfDirections": "interface.directions",
+  "IPSecStatus": "ipsec.status",
   "FlowDirection": "host.direction",
   "DnsErrno": "dns.errno",
   "DnsFlags": "dns.flags",

internal/pkg/helper/otel/otel_config_test.go

@@ -27,6 +27,7 @@ func TestOtelTransformConfig(t *testing.T) {
 		{Input: "DstK8S_HostName", Output: "destination.k8s.host.name", Multiplier: 0},
 		{Input: "DstK8S_Name", Output: "destination.k8s.name", Multiplier: 0},
 		{Input: "DstK8S_Namespace", Output: "destination.k8s.namespace.name", Multiplier: 0},
+		{Input: "DstK8S_NetworkName", Output: "destination.network.name", Multiplier: 0},
 		{Input: "DstK8S_OwnerName", Output: "destination.k8s.owner.name", Multiplier: 0},
 		{Input: "DstK8S_OwnerType", Output: "destination.k8s.owner.kind", Multiplier: 0},
 		{Input: "DstK8S_Type", Output: "destination.k8s.kind", Multiplier: 0},
@@ -36,6 +37,7 @@ func TestOtelTransformConfig(t *testing.T) {
 		{Input: "DstSubnetLabel", Output: "destination.subnet.label", Multiplier: 0},
 		{Input: "Flags", Output: "tcp.flags", Multiplier: 0},
 		{Input: "FlowDirection", Output: "host.direction", Multiplier: 0},
+		{Input: "IPSecStatus", Output: "ipsec.status", Multiplier: 0},
 		{Input: "IcmpCode", Output: "icmp.code", Multiplier: 0},
 		{Input: "IcmpType", Output: "icmp.type", Multiplier: 0},
 		{Input: "IfDirections", Output: "interface.directions", Multiplier: 0},
@@ -54,6 +56,7 @@ func TestOtelTransformConfig(t *testing.T) {
 		{Input: "SrcK8S_HostName", Output: "source.k8s.host.name", Multiplier: 0},
 		{Input: "SrcK8S_Name", Output: "source.k8s.name", Multiplier: 0},
 		{Input: "SrcK8S_Namespace", Output: "source.k8s.namespace.name", Multiplier: 0},
+		{Input: "SrcK8S_NetworkName", Output: "source.network.name", Multiplier: 0},
 		{Input: "SrcK8S_OwnerName", Output: "source.k8s.owner.name", Multiplier: 0},
 		{Input: "SrcK8S_OwnerType", Output: "source.k8s.owner.kind", Multiplier: 0},
 		{Input: "SrcK8S_Type", Output: "source.k8s.kind", Multiplier: 0},

internal/pkg/metrics/predefined_metrics.go

@@ -14,8 +14,6 @@ const (
 	tagNamespaces = "namespaces"
 	tagNodes      = "nodes"
 	tagWorkloads  = "workloads"
-	tagIngress    = "ingress"
-	tagEgress     = "egress"
 	tagBytes      = "bytes"
 	tagPackets    = "packets"
 )
@@ -25,7 +23,7 @@ var (
 	mapLabels      = map[string][]string{
 		tagNodes:      {"K8S_ClusterName", "SrcK8S_Zone", "DstK8S_Zone", "SrcK8S_HostName", "DstK8S_HostName"},
 		tagNamespaces: {"K8S_ClusterName", "SrcK8S_Zone", "DstK8S_Zone", "SrcK8S_Namespace", "DstK8S_Namespace", "K8S_FlowLayer", "SrcSubnetLabel", "DstSubnetLabel"},
-		tagWorkloads:  {"K8S_ClusterName", "SrcK8S_Zone", "DstK8S_Zone", "SrcK8S_Namespace", "DstK8S_Namespace", "K8S_FlowLayer", "SrcSubnetLabel", "DstSubnetLabel", "SrcK8S_OwnerName", "DstK8S_OwnerName", "SrcK8S_OwnerType", "DstK8S_OwnerType", "SrcK8S_Type", "DstK8S_Type"},
+		tagWorkloads:  {"K8S_ClusterName", "SrcK8S_Zone", "DstK8S_Zone", "SrcK8S_Namespace", "DstK8S_Namespace", "K8S_FlowLayer", "SrcSubnetLabel", "DstSubnetLabel", "SrcK8S_NetworkName", "DstK8S_NetworkName", "SrcK8S_OwnerName", "DstK8S_OwnerName", "SrcK8S_OwnerType", "DstK8S_OwnerType", "SrcK8S_Type", "DstK8S_Type"},
 	}
 	mapValueFields = map[string]string{
 		tagBytes:   "Bytes",
@@ -321,6 +319,9 @@ func GetDefinitions(fc *flowslatest.FlowCollectorSpec, allMetrics bool) []metric
 	if !fc.Processor.IsMultiClusterEnabled() {
 		labelsToRemove = append(labelsToRemove, "K8S_ClusterName")
 	}
+	if !fc.Agent.EBPF.IsUDNMappingEnabled() && !fc.Processor.HasSecondaryIndexes() {
+		labelsToRemove = append(labelsToRemove, "SrcK8S_NetworkName", "DstK8S_NetworkName")
+	}
 
 	var filterRecordType *metricslatest.MetricFilter
 	if fc.Processor.LogTypes != nil {

internal/pkg/metrics/predefined_metrics_test.go

@@ -67,7 +67,7 @@ func TestGetDefinitions(t *testing.T) {
 	assert.Equal([]string{"K8S_ClusterName", "SrcK8S_Zone", "DstK8S_Zone", "SrcK8S_Namespace", "DstK8S_Namespace", "K8S_FlowLayer", "SrcSubnetLabel", "DstSubnetLabel"}, res[1].Spec.Labels)
 	assert.Equal("workload_egress_packets_total", res[2].Spec.MetricName)
 	assert.Equal("Packets", res[2].Spec.ValueField)
-	assert.Equal([]string{"K8S_ClusterName", "SrcK8S_Zone", "DstK8S_Zone", "SrcK8S_Namespace", "DstK8S_Namespace", "K8S_FlowLayer", "SrcSubnetLabel", "DstSubnetLabel", "SrcK8S_OwnerName", "DstK8S_OwnerName", "SrcK8S_OwnerType", "DstK8S_OwnerType", "SrcK8S_Type", "DstK8S_Type"}, res[2].Spec.Labels)
+	assert.Equal([]string{"K8S_ClusterName", "SrcK8S_Zone", "DstK8S_Zone", "SrcK8S_Namespace", "DstK8S_Namespace", "K8S_FlowLayer", "SrcSubnetLabel", "DstSubnetLabel", "SrcK8S_NetworkName", "DstK8S_NetworkName", "SrcK8S_OwnerName", "DstK8S_OwnerName", "SrcK8S_OwnerType", "DstK8S_OwnerType", "SrcK8S_Type", "DstK8S_Type"}, res[2].Spec.Labels)
 }
 
 func TestGetDefinitionsRemoveZoneCluster(t *testing.T) {
@@ -86,5 +86,138 @@ func TestGetDefinitionsRemoveZoneCluster(t *testing.T) {
 	assert.Equal([]string{"SrcK8S_Namespace", "DstK8S_Namespace", "K8S_FlowLayer", "SrcSubnetLabel", "DstSubnetLabel"}, res[1].Spec.Labels)
 	assert.Equal("workload_egress_packets_total", res[2].Spec.MetricName)
 	assert.Equal("Packets", res[2].Spec.ValueField)
-	assert.Equal([]string{"SrcK8S_Namespace", "DstK8S_Namespace", "K8S_FlowLayer", "SrcSubnetLabel", "DstSubnetLabel", "SrcK8S_OwnerName", "DstK8S_OwnerName", "SrcK8S_OwnerType", "DstK8S_OwnerType", "SrcK8S_Type", "DstK8S_Type"}, res[2].Spec.Labels)
+	assert.Equal([]string{"SrcK8S_Namespace", "DstK8S_Namespace", "K8S_FlowLayer", "SrcSubnetLabel", "DstSubnetLabel", "SrcK8S_NetworkName", "DstK8S_NetworkName", "SrcK8S_OwnerName", "DstK8S_OwnerName", "SrcK8S_OwnerType", "DstK8S_OwnerType", "SrcK8S_Type", "DstK8S_Type"}, res[2].Spec.Labels)
+}
+
+func TestGetDefinitionsRemoveNetworkLabels(t *testing.T) {
+	assert := assert.New(t)
+
+	spec := util.SpecForMetrics("workload_ingress_bytes_total")
+	// Disable multiNetworks feature (UDN mapping and secondary indexes)
+	spec.Agent.EBPF.Features = []flowslatest.AgentFeature{flowslatest.FlowRTT, flowslatest.DNSTracking, flowslatest.PacketDrop} // Remove UDNMapping if it was there
+	spec.Processor.Advanced = nil                                                                                               // Ensure no secondary indexes
+	res := GetDefinitions(spec, false)
+	assert.Len(res, 1)
+	// Workload metric should have network labels removed
+	assert.Equal("workload_ingress_bytes_total", res[0].Spec.MetricName)
+	assert.Equal([]string{"K8S_ClusterName", "SrcK8S_Zone", "DstK8S_Zone", "SrcK8S_Namespace", "DstK8S_Namespace", "K8S_FlowLayer", "SrcSubnetLabel", "DstSubnetLabel", "SrcK8S_OwnerName", "DstK8S_OwnerName", "SrcK8S_OwnerType", "DstK8S_OwnerType", "SrcK8S_Type", "DstK8S_Type"}, res[0].Spec.Labels)
+}
+
+func TestGetDefinitionsNodeMetrics(t *testing.T) {
+	assert := assert.New(t)
+
+	res := GetDefinitions(util.SpecForMetrics("node_ingress_bytes_total", "node_egress_packets_total", "node_flows_total"), false)
+	assert.Len(res, 3)
+	assert.Equal("node_ingress_bytes_total", res[0].Spec.MetricName)
+	assert.Equal("Bytes", res[0].Spec.ValueField)
+	assert.Equal([]string{"K8S_ClusterName", "SrcK8S_Zone", "DstK8S_Zone", "SrcK8S_HostName", "DstK8S_HostName"}, res[0].Spec.Labels)
+	assert.Equal("node_egress_packets_total", res[1].Spec.MetricName)
+	assert.Equal("Packets", res[1].Spec.ValueField)
+	assert.Equal([]string{"K8S_ClusterName", "SrcK8S_Zone", "DstK8S_Zone", "SrcK8S_HostName", "DstK8S_HostName"}, res[1].Spec.Labels)
+	assert.Equal("node_flows_total", res[2].Spec.MetricName)
+	assert.Empty(res[2].Spec.ValueField)
+	assert.Equal([]string{"K8S_ClusterName", "SrcK8S_Zone", "DstK8S_Zone", "SrcK8S_HostName", "DstK8S_HostName"}, res[2].Spec.Labels)
+}
+
+func TestGetDefinitionsNamespaceMetrics(t *testing.T) {
+	assert := assert.New(t)
+
+	res := GetDefinitions(util.SpecForMetrics("namespace_ingress_bytes_total", "namespace_egress_packets_total", "namespace_flows_total"), false)
+	assert.Len(res, 3)
+	assert.Equal("namespace_ingress_bytes_total", res[0].Spec.MetricName)
+	assert.Equal("Bytes", res[0].Spec.ValueField)
+	assert.Equal([]string{"K8S_ClusterName", "SrcK8S_Zone", "DstK8S_Zone", "SrcK8S_Namespace", "DstK8S_Namespace", "K8S_FlowLayer", "SrcSubnetLabel", "DstSubnetLabel"}, res[0].Spec.Labels)
+	assert.Equal("namespace_egress_packets_total", res[1].Spec.MetricName)
+	assert.Equal("Packets", res[1].Spec.ValueField)
+	assert.Equal([]string{"K8S_ClusterName", "SrcK8S_Zone", "DstK8S_Zone", "SrcK8S_Namespace", "DstK8S_Namespace", "K8S_FlowLayer", "SrcSubnetLabel", "DstSubnetLabel"}, res[1].Spec.Labels)
+	assert.Equal("namespace_flows_total", res[2].Spec.MetricName)
+	assert.Empty(res[2].Spec.ValueField)
+	assert.Equal([]string{"K8S_ClusterName", "SrcK8S_Zone", "DstK8S_Zone", "SrcK8S_Namespace", "DstK8S_Namespace", "K8S_FlowLayer", "SrcSubnetLabel", "DstSubnetLabel"}, res[2].Spec.Labels)
+}
+
+func TestGetDefinitionsWorkloadMetrics(t *testing.T) {
+	assert := assert.New(t)
+
+	res := GetDefinitions(util.SpecForMetrics("workload_ingress_bytes_total", "workload_egress_packets_total", "workload_flows_total"), false)
+	assert.Len(res, 3)
+	assert.Equal("workload_ingress_bytes_total", res[0].Spec.MetricName)
+	assert.Equal("Bytes", res[0].Spec.ValueField)
+	assert.Equal([]string{"K8S_ClusterName", "SrcK8S_Zone", "DstK8S_Zone", "SrcK8S_Namespace", "DstK8S_Namespace", "K8S_FlowLayer", "SrcSubnetLabel", "DstSubnetLabel", "SrcK8S_NetworkName", "DstK8S_NetworkName", "SrcK8S_OwnerName", "DstK8S_OwnerName", "SrcK8S_OwnerType", "DstK8S_OwnerType", "SrcK8S_Type", "DstK8S_Type"}, res[0].Spec.Labels)
+	assert.Equal("workload_egress_packets_total", res[1].Spec.MetricName)
+	assert.Equal("Packets", res[1].Spec.ValueField)
+	assert.Equal("workload_flows_total", res[2].Spec.MetricName)
+	assert.Empty(res[2].Spec.ValueField)
+}
+
+func TestGetDefinitionsAllMetricTypesForGroup(t *testing.T) {
+	assert := assert.New(t)
+
+	// Test all metric types for a single group (node)
+	res := GetDefinitions(util.SpecForMetrics("node_ingress_bytes_total", "node_rtt_seconds", "node_drop_packets_total", "node_dns_latency_seconds", "node_ipsec_flows_total"), false)
+	assert.Len(res, 5)
+
+	// Check that different metric types are present
+	metricNames := make([]string, len(res))
+	for i, m := range res {
+		metricNames[i] = m.Spec.MetricName
+	}
+	assert.Contains(metricNames, "node_ingress_bytes_total")
+	assert.Contains(metricNames, "node_rtt_seconds")
+	assert.Contains(metricNames, "node_drop_packets_total")
+	assert.Contains(metricNames, "node_dns_latency_seconds")
+	assert.Contains(metricNames, "node_ipsec_flows_total")
+
+	// Verify RTT has correct configuration
+	for _, m := range res {
+		if m.Spec.MetricName == "node_rtt_seconds" {
+			assert.Equal("TimeFlowRttNs", m.Spec.ValueField)
+			assert.Equal("1000000000", m.Spec.Divider)
+			assert.Len(m.Spec.Filters, 1)
+		}
+	}
+}
+
+func TestGetDefinitionsMixedGroups(t *testing.T) {
+	assert := assert.New(t)
+
+	// Test requesting metrics from different groups
+	res := GetDefinitions(util.SpecForMetrics("node_ingress_bytes_total", "namespace_flows_total", "workload_egress_packets_total"), false)
+	assert.Len(res, 3)
+
+	metricNames := make([]string, len(res))
+	for i, m := range res {
+		metricNames[i] = m.Spec.MetricName
+	}
+	assert.Contains(metricNames, "node_ingress_bytes_total")
+	assert.Contains(metricNames, "namespace_flows_total")
+	assert.Contains(metricNames, "workload_egress_packets_total")
+}
+
+func TestGetDefinitionsRemoveZoneLabels(t *testing.T) {
+	assert := assert.New(t)
+
+	spec := util.SpecForMetrics("node_ingress_bytes_total", "namespace_flows_total")
+	spec.Processor.AddZone = ptr.To(false)
+	res := GetDefinitions(spec, false)
+	assert.Len(res, 2)
+
+	// All metrics should have zone labels removed
+	for _, m := range res {
+		assert.NotContains(m.Spec.Labels, "SrcK8S_Zone")
+		assert.NotContains(m.Spec.Labels, "DstK8S_Zone")
+	}
+}
+
+func TestGetDefinitionsRemoveMultiClusterLabels(t *testing.T) {
+	assert := assert.New(t)
+
+	spec := util.SpecForMetrics("node_ingress_bytes_total", "namespace_flows_total")
+	spec.Processor.MultiClusterDeployment = ptr.To(false)
+	res := GetDefinitions(spec, false)
+	assert.Len(res, 2)
+
+	// All metrics should have cluster label removed
+	for _, m := range res {
+		assert.NotContains(m.Spec.Labels, "K8S_ClusterName")
+	}
 }

internal/pkg/test/util/utils.go

@@ -10,7 +10,7 @@ func SpecForMetrics(metrics ...string) *flowslatest.FlowCollectorSpec {
 		Agent: flowslatest.FlowCollectorAgent{
 			EBPF: flowslatest.FlowCollectorEBPF{
 				Privileged: true,
-				Features:   []flowslatest.AgentFeature{flowslatest.FlowRTT, flowslatest.DNSTracking, flowslatest.PacketDrop},
+				Features:   []flowslatest.AgentFeature{flowslatest.FlowRTT, flowslatest.DNSTracking, flowslatest.PacketDrop, flowslatest.UDNMapping, flowslatest.IPSec},
 			},
 		},
 		Processor: flowslatest.FlowCollectorFLP{