From f23b7b3b4be08f008d82c1faecbe1cb3732c3c45 Mon Sep 17 00:00:00 2001 From: 08volt Date: Thu, 10 Oct 2024 14:37:37 +0000 Subject: [PATCH] feature(l4): Add zonal affinity support and related controls This commit implements zonal affinity capabilities for L4 Internal Load Balancers: - Add zonal affinity flag for L4 ILB controllers - Implement backendService zonal affinity control logic - Add backendService minimum API version control - Update backendService tests to include zonal affinity cases with expected NetworkPassThroughLbTrafficPolicy - Add handler for service spec Spec.TrafficDistribution change events - Add L4 TestZonalAffinity integration tests --- cmd/glbc/main.go | 1 + pkg/backends/backends.go | 98 +++++++++++++++++++++++++++-- pkg/backends/backends_test.go | 99 +++++++++++++++++++++++++++++ pkg/context/context.go | 1 + pkg/flags/flags.go | 2 + pkg/l4lb/l4controller.go | 7 +++ pkg/loadbalancers/l4.go | 13 ++++ pkg/loadbalancers/l4_test.go | 101 ++++++++++++++++++++++++++++++ pkg/loadbalancers/l4netlb_test.go | 2 +- 9 files changed, 319 insertions(+), 5 deletions(-) diff --git a/cmd/glbc/main.go b/cmd/glbc/main.go index 7b92b9b8a5..4ecd5daad5 100644 --- a/cmd/glbc/main.go +++ b/cmd/glbc/main.go @@ -334,6 +334,7 @@ func main() { EnableL4NetLBNEGsDefault: flags.F.EnableL4NetLBNEGDefault, EnableL4ILBMixedProtocol: flags.F.EnableL4ILBMixedProtocol, EnableL4NetLBMixedProtocol: flags.F.EnableL4NetLBMixedProtocol, + EnableZonalAffinity: flags.F.EnableZonalAffinity, } ctx, err := ingctx.NewControllerContext(kubeClient, backendConfigClient, frontendConfigClient, firewallCRClient, svcNegClient, svcAttachmentClient, networkClient, nodeTopologyClient, eventRecorderKubeClient, cloud, namer, kubeSystemUID, ctxConfig, rootLogger) if err != nil { diff --git a/pkg/backends/backends.go b/pkg/backends/backends.go index fe33100244..d44bee9776 100644 --- a/pkg/backends/backends.go +++ b/pkg/backends/backends.go @@ -35,6 +35,9 @@ const ( DefaultConnectionDrainingTimeoutSeconds = 30 defaultTrackingMode = "PER_CONNECTION" PerSessionTrackingMode = "PER_SESSION" // the only one supported with strong session affinity + DefaultZonalAffinitySpillover = "ZONAL_AFFINITY_SPILL_CROSS_ZONE" + DefaultZonalAffinitySpilloverRatio = 0 + ZonalAffinityDisabledSpillover = "ZONAL_AFFINITY_DISABLED" ) // LocalityLBPolicyType is the type of locality lb policy the backend service should use. @@ -90,6 +93,31 @@ type L4BackendServiceParams struct { NetworkInfo *network.NetworkInfo ConnectionTrackingPolicy *composite.BackendServiceConnectionTrackingPolicy LocalityLbPolicy LocalityLBPolicyType + EnableZonalAffinity bool +} + +var versionPrecedence = map[meta.Version]int{ + meta.VersionAlpha: 2, + meta.VersionBeta: 1, + meta.VersionGA: 0, +} + +// maxVersion returns the higher version based on precedence. +func maxVersion(a, b meta.Version) meta.Version { + precedenceA, okA := versionPrecedence[a] + if !okA { + precedenceA = 0 // Use VersionGA precedence if invalid. + } + + precedenceB, okB := versionPrecedence[b] + if !okB { + precedenceB = 0 // Use VersionGA precedence if invalid. + } + + if precedenceA > precedenceB { + return a + } + return b } // ensureDescription updates the BackendService Description with the expected value @@ -327,6 +355,14 @@ func (p *Pool) DeleteSignedURLKey(be *composite.BackendService, keyName string, return nil } +// minRequiredVersion to create a backend service with the given params +func minRequiredVersion(params L4BackendServiceParams) meta.Version { + if params.EnableZonalAffinity { + return meta.VersionAlpha + } + return meta.VersionGA +} + // EnsureL4BackendService creates or updates the backend service with the given name. func (p *Pool) EnsureL4BackendService(params L4BackendServiceParams, beLogger klog.Logger) (*composite.BackendService, utils.ResourceSyncStatus, error) { start := time.Now() @@ -342,11 +378,14 @@ func (p *Pool) EnsureL4BackendService(params L4BackendServiceParams, beLogger kl if err != nil { return nil, utils.ResourceResync, err } + currentBS, err := composite.GetBackendService(p.cloud, key, meta.VersionGA, beLogger) if err != nil && !utils.IsNotFoundError(err) { return nil, utils.ResourceResync, err } - desc, err := utils.MakeL4LBServiceDescription(params.NamespacedName.String(), "", meta.VersionGA, false, utils.ILB) + + expectedVersion := minRequiredVersion(params) + expectedDesc, err := utils.MakeL4LBServiceDescription(params.NamespacedName.String(), "", expectedVersion, false, utils.ILB) if err != nil { beLogger.Info("EnsureL4BackendService: Failed to generate description for BackendService", "err", err) } @@ -354,13 +393,19 @@ func (p *Pool) EnsureL4BackendService(params L4BackendServiceParams, beLogger kl expectedBS := &composite.BackendService{ Name: params.Name, Protocol: params.Protocol, - Description: desc, + Version: expectedVersion, + Description: expectedDesc, HealthChecks: []string{params.HealthCheckLink}, SessionAffinity: utils.TranslateAffinityType(params.SessionAffinity, beLogger), LoadBalancingScheme: params.Scheme, LocalityLbPolicy: string(params.LocalityLbPolicy), } + if params.EnableZonalAffinity { + beLogger.V(2).Info("EnsureL4BackendService: using Zonal Affinity", "spillover", DefaultZonalAffinitySpillover, "spilloverRatio", DefaultZonalAffinitySpilloverRatio) + expectedBS.NetworkPassThroughLbTrafficPolicy = defaultZonalAffinityTrafficPolicy() + } + // We need this configuration only for Strong Session Affinity feature if p.useConnectionTrackingPolicy { beLogger.V(2).Info(fmt.Sprintf("EnsureL4BackendService: using connection tracking policy: %+v", params.ConnectionTrackingPolicy)) @@ -388,7 +433,7 @@ func (p *Pool) EnsureL4BackendService(params L4BackendServiceParams, beLogger kl // We need to perform a GCE call to re-fetch the object we just created // so that the "Fingerprint" field is filled in. This is needed to update the // object without error. The lookup is also needed to populate the selfLink. - createdBS, err := composite.GetBackendService(p.cloud, key, meta.VersionGA, beLogger) + createdBS, err := composite.GetBackendService(p.cloud, key, expectedBS.Version, beLogger) return createdBS, utils.ResourceUpdate, err } else { // TODO(FelipeYepez) remove this check once LocalityLBPolicyMaglev does not require allow lisiting @@ -398,6 +443,17 @@ func (p *Pool) EnsureL4BackendService(params L4BackendServiceParams, beLogger kl expectedBS.LocalityLbPolicy = string(LocalityLBPolicyMaglev) } + + // Use the version with most priority if the BackendService was already using one + currentVersion := meta.VersionGA + var currentDesc utils.L4LBResourceDescription + err = currentDesc.Unmarshal(currentBS.Description) + if err != nil { + beLogger.V(0).Error(err, "EnsureL4BackendService: error unmarshaling backend service description") + } else { + currentVersion = currentDesc.APIVersion + } + expectedBS.Version = maxVersion(currentVersion, expectedBS.Version) } if backendSvcEqual(expectedBS, currentBS, p.useConnectionTrackingPolicy) { @@ -418,7 +474,7 @@ func (p *Pool) EnsureL4BackendService(params L4BackendServiceParams, beLogger kl } beLogger.V(2).Info("EnsureL4BackendService: updated backend service successfully") - updatedBS, err := composite.GetBackendService(p.cloud, key, meta.VersionGA, beLogger) + updatedBS, err := composite.GetBackendService(p.cloud, key, expectedBS.Version, beLogger) return updatedBS, utils.ResourceUpdate, err } @@ -447,9 +503,34 @@ func backendSvcEqual(newBS, oldBS *composite.BackendService, compareConnectionTr (newBS.LocalityLbPolicy == string(LocalityLBPolicyDefault) && oldBS.LocalityLbPolicy == string(LocalityLBPolicyMaglev)) || (newBS.LocalityLbPolicy == string(LocalityLBPolicyMaglev) && oldBS.LocalityLbPolicy == string(LocalityLBPolicyDefault))) + // If zonal affinity is set, needs to be equal + svcsEqual = svcsEqual && zonalAffinityEqual(newBS, oldBS) return svcsEqual } +func convertNetworkLbTrafficPolicyToZonalAffinity(trafficPolicy *composite.BackendServiceNetworkPassThroughLbTrafficPolicy) composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity { + if trafficPolicy == nil || trafficPolicy.ZonalAffinity == nil { + return composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{ + Spillover: ZonalAffinityDisabledSpillover, + SpilloverRatio: 0, + } + } + + return *trafficPolicy.ZonalAffinity +} + +func zonalAffinityEqual(a, b *composite.BackendService) bool { + aZonalAffinity := convertNetworkLbTrafficPolicyToZonalAffinity(a.NetworkPassThroughLbTrafficPolicy) + bZonalAffinity := convertNetworkLbTrafficPolicyToZonalAffinity(b.NetworkPassThroughLbTrafficPolicy) + + // Compare Spillover values + spilloverEqual := aZonalAffinity.Spillover == bZonalAffinity.Spillover + // Compare SpilloverRatio values + spilloverRatioEqual := aZonalAffinity.SpilloverRatio == bZonalAffinity.SpilloverRatio + + return spilloverEqual && spilloverRatioEqual +} + // connectionTrackingPolicyEqual returns true if both elements are equal // and return false if at least one parameter is different func connectionTrackingPolicyEqual(a, b *composite.BackendServiceConnectionTrackingPolicy) bool { @@ -460,3 +541,12 @@ func connectionTrackingPolicyEqual(a, b *composite.BackendServiceConnectionTrack a.EnableStrongAffinity == b.EnableStrongAffinity && a.IdleTimeoutSec == b.IdleTimeoutSec } + +func defaultZonalAffinityTrafficPolicy() *composite.BackendServiceNetworkPassThroughLbTrafficPolicy { + return &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{ + ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{ + Spillover: DefaultZonalAffinitySpillover, + SpilloverRatio: DefaultZonalAffinitySpilloverRatio, + }, + } +} diff --git a/pkg/backends/backends_test.go b/pkg/backends/backends_test.go index aa6dfea5fb..af1efc4d43 100644 --- a/pkg/backends/backends_test.go +++ b/pkg/backends/backends_test.go @@ -626,6 +626,105 @@ func TestBackendSvcEqual(t *testing.T) { }, wantEqual: false, }, + { + desc: "Test existing backend service diff with zonal affinity feature enabled", + oldBackendService: &composite.BackendService{ + NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{ + ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{ + Spillover: "ZONAL_AFFINITY_SPILL_CROSS_ZONE", + SpilloverRatio: 0.7, + }, + }, + }, + newBackendService: &composite.BackendService{ + NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{ + ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{ + Spillover: "ZONAL_AFFINITY_SPILL_CROSS_ZONE", + SpilloverRatio: 0.7, + }, + }, + }, + wantEqual: true, + }, + { + desc: "Test existing backend service diff with zonal affinity feature enabled but different ratio", + oldBackendService: &composite.BackendService{ + NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{ + ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{ + Spillover: "ZONAL_AFFINITY_SPILL_CROSS_ZONE", + SpilloverRatio: 0.7, + }, + }, + }, + newBackendService: &composite.BackendService{ + NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{ + ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{ + Spillover: "ZONAL_AFFINITY_SPILL_CROSS_ZONE", + SpilloverRatio: 0.3, + }, + }, + }, + wantEqual: false, + }, + { + desc: "Test existing backend service diff with zonal affinity feature enabled but different spillover strategy", + oldBackendService: &composite.BackendService{ + NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{ + ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{ + Spillover: "ZONAL_AFFINITY_STAY_WITHIN_ZONE", + SpilloverRatio: 0.3, + }, + }, + }, + newBackendService: &composite.BackendService{ + NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{ + ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{ + Spillover: "ZONAL_AFFINITY_SPILL_CROSS_ZONE", + SpilloverRatio: 0.3, + }, + }, + }, + wantEqual: false, + }, + { + desc: "Test existing backend service diff enabling zonal affinity feature", + oldBackendService: &composite.BackendService{}, + newBackendService: &composite.BackendService{ + NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{ + ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{ + Spillover: "ZONAL_AFFINITY_SPILL_CROSS_ZONE", + SpilloverRatio: 0.3, + }, + }, + }, + wantEqual: false, + }, + { + desc: "Test existing backend service diff enabling zonal affinity feature", + oldBackendService: &composite.BackendService{ + NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{ + ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{ + Spillover: "ZONAL_AFFINITY_SPILL_CROSS_ZONE", + SpilloverRatio: 0.3, + }, + }, + }, + newBackendService: &composite.BackendService{}, + wantEqual: false, + }, + { + desc: "Test existing backend service diff with zonal affinity disables", + oldBackendService: &composite.BackendService{ + NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{ + ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{ + Spillover: "ZONAL_AFFINITY_DISABLED", + SpilloverRatio: 0, + }, + }, + }, + newBackendService: &composite.BackendService{}, + wantEqual: true, + }, } { tc := tc t.Run(tc.desc, func(t *testing.T) { diff --git a/pkg/context/context.go b/pkg/context/context.go index 994bd08490..772ed5367a 100644 --- a/pkg/context/context.go +++ b/pkg/context/context.go @@ -133,6 +133,7 @@ type ControllerContextConfig struct { EnableIngressRegionalExternal bool EnableWeightedL4ILB bool EnableWeightedL4NetLB bool + EnableZonalAffinity bool DisableL4LBFirewall bool EnableL4NetLBNEGs bool EnableL4NetLBNEGsDefault bool diff --git a/pkg/flags/flags.go b/pkg/flags/flags.go index f747d14d79..55e249fe28 100644 --- a/pkg/flags/flags.go +++ b/pkg/flags/flags.go @@ -139,6 +139,7 @@ var F = struct { EnableWeightedL4NetLB bool EnableDiscretePortForwarding bool EnableMultiProjectMode bool + EnableZonalAffinity bool ProviderConfigNameLabelKey string EnableL4ILBMixedProtocol bool EnableL4NetLBMixedProtocol bool @@ -326,6 +327,7 @@ L7 load balancing. CSV values accepted. Example: -node-port-ranges=80,8080,400-5 flag.StringVar(&F.NodeTopologyCRName, "node-topology-cr-name", "default", "The name of the Node Topology CR.") flag.BoolVar(&F.EnableWeightedL4ILB, "enable-weighted-l4-ilb", false, "Enable Weighted Load balancing for L4 ILB.") flag.BoolVar(&F.EnableWeightedL4NetLB, "enable-weighted-l4-netlb", false, "EnableWeighted Load balancing for L4 NetLB .") + flag.BoolVar(&F.EnableZonalAffinity, "enable-zonal-affinity", false, "Enable Zonal Affinity for L4 ILB.") flag.Float32Var(&F.KubeClientQPS, "kube-client-qps", 0.0, "The QPS that the controllers' kube client should adhere to through client side throttling. If zero, client will be created with default settings.") flag.IntVar(&F.KubeClientBurst, "kube-client-burst", 0, "The burst QPS that the controllers' kube client should adhere to through client side throttling. If zero, client will be created with default settings.") flag.BoolVar(&F.EnableDiscretePortForwarding, "enable-discrete-port-forwarding", false, "Enable forwarding of individual ports instead of port ranges.") diff --git a/pkg/l4lb/l4controller.go b/pkg/l4lb/l4controller.go index 765b6b5170..02080055d2 100644 --- a/pkg/l4lb/l4controller.go +++ b/pkg/l4lb/l4controller.go @@ -287,6 +287,7 @@ func (l4c *L4Controller) processServiceCreateOrUpdate(service *v1.Service, svcLo EnableWeightedLB: l4c.ctx.EnableWeightedL4ILB, DisableNodesFirewallProvisioning: l4c.ctx.DisableL4LBFirewall, EnableMixedProtocol: l4c.ctx.EnableL4ILBMixedProtocol, + EnableZonalAffinity: l4c.ctx.EnableZonalAffinity, } l4 := loadbalancers.NewL4Handler(l4ilbParams, svcLogger) syncResult := l4.EnsureInternalLoadBalancer(utils.GetNodeNames(nodes), service) @@ -369,6 +370,7 @@ func (l4c *L4Controller) processServiceDeletion(key string, svc *v1.Service, svc EnableWeightedLB: l4c.ctx.EnableWeightedL4ILB, DisableNodesFirewallProvisioning: l4c.ctx.DisableL4LBFirewall, EnableMixedProtocol: l4c.ctx.EnableL4ILBMixedProtocol, + EnableZonalAffinity: l4c.ctx.EnableZonalAffinity, } l4 := loadbalancers.NewL4Handler(l4ilbParams, svcLogger) l4c.ctx.Recorder(svc.Namespace).Eventf(svc, v1.EventTypeNormal, "DeletingLoadBalancer", "Deleting load balancer for %s", key) @@ -579,6 +581,11 @@ func (l4c *L4Controller) needsUpdate(oldService *v1.Service, newService *v1.Serv oldService.Spec.HealthCheckNodePort, newService.Spec.HealthCheckNodePort) return true } + if oldService.Spec.TrafficDistribution != newService.Spec.TrafficDistribution { + recorder.Eventf(newService, v1.EventTypeNormal, "TrafficDistribution", "%v -> %v", + oldService.Spec.TrafficDistribution, newService.Spec.TrafficDistribution) + return true + } if l4c.enableDualStack && !reflect.DeepEqual(oldService.Spec.IPFamilies, newService.Spec.IPFamilies) { recorder.Eventf(newService, v1.EventTypeNormal, "IPFamilies", "%v -> %v", oldService.Spec.IPFamilies, newService.Spec.IPFamilies) diff --git a/pkg/loadbalancers/l4.go b/pkg/loadbalancers/l4.go index 66975b54a9..6b8f3fcbb8 100644 --- a/pkg/loadbalancers/l4.go +++ b/pkg/loadbalancers/l4.go @@ -46,6 +46,7 @@ import ( const ( subnetInternalIPv6AccessType = "INTERNAL" + trafficDistribuitionZonalAffinity = "PreferClose" WeightedLBPodsPerNodeAllowlistMessage = "Weighted Load Balancing for L4 " + "Internal Passthrough Load Balancers requires project allowlisting. If " + "you need access to this feature please contact Google Cloud support team" @@ -73,6 +74,7 @@ type L4 struct { enableWeightedLB bool enableMixedProtocol bool disableNodesFirewallProvisioning bool + enableZonalAffinity bool svcLogger klog.Logger } @@ -110,6 +112,7 @@ type L4ILBParams struct { DualStackEnabled bool NetworkResolver network.Resolver EnableWeightedLB bool + EnableZonalAffinity bool DisableNodesFirewallProvisioning bool EnableMixedProtocol bool } @@ -132,6 +135,7 @@ func NewL4Handler(params *L4ILBParams, logger klog.Logger) *L4 { enableWeightedLB: params.EnableWeightedLB, enableMixedProtocol: params.EnableMixedProtocol, disableNodesFirewallProvisioning: params.DisableNodesFirewallProvisioning, + enableZonalAffinity: params.EnableZonalAffinity, svcLogger: logger, } l4.NamespacedName = types.NamespacedName{Name: params.Service.Name, Namespace: params.Service.Namespace} @@ -532,6 +536,8 @@ func (l4 *L4) EnsureInternalLoadBalancer(nodeNames []string, svc *corev1.Service localityLbPolicy := l4.determineBackendServiceLocalityPolicy() + enableZonalAffinity := l4.requireZonalAffinity(svc) + // ensure backend service backendParams := backends.L4BackendServiceParams{ Name: bsName, @@ -542,6 +548,7 @@ func (l4 *L4) EnsureInternalLoadBalancer(nodeNames []string, svc *corev1.Service NamespacedName: l4.NamespacedName, NetworkInfo: &l4.network, ConnectionTrackingPolicy: noConnectionTrackingPolicy, + EnableZonalAffinity: enableZonalAffinity, LocalityLbPolicy: localityLbPolicy, } bs, bsSyncStatus, err := l4.backendPool.EnsureL4BackendService(backendParams, l4.svcLogger) @@ -583,6 +590,12 @@ func (l4 *L4) EnsureInternalLoadBalancer(nodeNames []string, svc *corev1.Service return result } +func (l4 *L4) requireZonalAffinity(svc *corev1.Service) bool { + return l4.enableZonalAffinity && // zonal affinity flag is enabled + svc.Spec.TrafficDistribution != nil && // traffic distribution field is set + *svc.Spec.TrafficDistribution == trafficDistribuitionZonalAffinity // traffic distribution field is set to zonal affinity default "PreferClose" +} + func (l4 *L4) provideHealthChecks(nodeNames []string, result *L4ILBSyncResult) string { if l4.enableDualStack { return l4.provideDualStackHealthChecks(nodeNames, result) diff --git a/pkg/loadbalancers/l4_test.go b/pkg/loadbalancers/l4_test.go index acbcda58fd..89052e389b 100644 --- a/pkg/loadbalancers/l4_test.go +++ b/pkg/loadbalancers/l4_test.go @@ -2388,6 +2388,107 @@ func TestWeightedILB(t *testing.T) { } } +func TestZonalAffinity(t *testing.T) { + t.Parallel() + + zonalAffinityTrafficDistributionKey := "PreferClose" + + tests := []struct { + desc string + addAnnotationForZonalAffinity bool + zonalAffinityFlagEnabled bool + trafficDistributionSpec *string + wantZonalAffinityEnabled bool + wantZonalAffinitySpilloverRatio float64 + }{ + { + desc: "Flag enabled, Service with TrafficDistribution Spec", + zonalAffinityFlagEnabled: true, + trafficDistributionSpec: &zonalAffinityTrafficDistributionKey, + wantZonalAffinityEnabled: true, + wantZonalAffinitySpilloverRatio: 0, + }, + { + desc: "Flag enabled, Service without TrafficDistribution Spec", + zonalAffinityFlagEnabled: true, + trafficDistributionSpec: nil, + wantZonalAffinityEnabled: false, + wantZonalAffinitySpilloverRatio: 0, + }, + { + desc: "Flag DISABLED, Service with TrafficDistribution Spec", + zonalAffinityFlagEnabled: false, + trafficDistributionSpec: &zonalAffinityTrafficDistributionKey, + wantZonalAffinityEnabled: false, + wantZonalAffinitySpilloverRatio: 0, + }, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.desc, func(t *testing.T) { + // t.Parallel() + + svc := test.NewL4ILBService(false, 8080) + + svc.Spec.TrafficDistribution = tc.trafficDistributionSpec + + nodeNames := []string{"test-node-1"} + vals := gce.DefaultTestClusterValues() + fakeGCE := getFakeGCECloud(vals) + + namer := namer_util.NewL4Namer(kubeSystemUID, nil) + + networkInfo := network.DefaultNetwork(fakeGCE) + + l4ilbParams := &L4ILBParams{ + Service: svc, + Cloud: fakeGCE, + Namer: namer, + Recorder: record.NewFakeRecorder(100), + NetworkResolver: network.NewFakeResolver(networkInfo), + EnableZonalAffinity: tc.zonalAffinityFlagEnabled, + } + l4 := NewL4Handler(l4ilbParams, klog.TODO()) + l4.healthChecks = healthchecksl4.Fake(fakeGCE, l4ilbParams.Recorder) + + if _, err := test.CreateAndInsertNodes(l4.cloud, nodeNames, vals.ZoneName); err != nil { + t.Errorf("Unexpected error when adding nodes %v", err) + } + + result := l4.EnsureInternalLoadBalancer(nodeNames, svc) + if result.Error != nil { + t.Fatalf("Failed to ensure internal loadBalancer, err %v", result.Error) + } + backendServiceName := l4.namer.L4Backend(l4.Service.Namespace, l4.Service.Name) + key := meta.RegionalKey(backendServiceName, l4.cloud.Region()) + bs, err := composite.GetBackendService(l4.cloud, key, meta.VersionAlpha, klog.TODO()) + if err != nil { + t.Fatalf("failed to read BackendService, %v", err) + } + + if bs.NetworkPassThroughLbTrafficPolicy != nil && !tc.wantZonalAffinityEnabled { + t.Fatalf("Unexpected BackendService ZonalAffinity, got value, wanted nil") + } + + if tc.wantZonalAffinityEnabled { + if bs.NetworkPassThroughLbTrafficPolicy == nil || bs.NetworkPassThroughLbTrafficPolicy.ZonalAffinity == nil { + t.Fatalf("Unexpected BackendService ZonalAffinity, got nil") + } + + if bs.NetworkPassThroughLbTrafficPolicy.ZonalAffinity.Spillover != backends.DefaultZonalAffinitySpillover { + t.Errorf("Unexpected BackendService ZonalAffinity Spillover: got %q, want %q", bs.NetworkPassThroughLbTrafficPolicy.ZonalAffinity.Spillover, backends.DefaultZonalAffinitySpillover) + } + + if bs.NetworkPassThroughLbTrafficPolicy.ZonalAffinity.SpilloverRatio != tc.wantZonalAffinitySpilloverRatio { + t.Errorf("Unexpected BackendService ZonalAffinity SpilloverRatio: got %v, want %v", bs.NetworkPassThroughLbTrafficPolicy.ZonalAffinity.SpilloverRatio, tc.wantZonalAffinitySpilloverRatio) + } + } + }) + } + +} + func TestDisableILBIngressFirewall(t *testing.T) { t.Parallel() fakeGCE := getFakeGCECloud(gce.DefaultTestClusterValues()) diff --git a/pkg/loadbalancers/l4netlb_test.go b/pkg/loadbalancers/l4netlb_test.go index 2400606d90..350d6dba84 100644 --- a/pkg/loadbalancers/l4netlb_test.go +++ b/pkg/loadbalancers/l4netlb_test.go @@ -1354,7 +1354,7 @@ func assertNetLBResources(t *testing.T, l4NetLB *L4NetLB, nodeNames []string) { backendService, err := getAndVerifyNetLBBackendService(l4NetLB, healthcheck) if err != nil { - t.Errorf("getAndVerifyNetLBBackendService(_, %v) returned error %v, want nil", healthcheck, err) + t.Fatalf("getAndVerifyNetLBBackendService(_, %v) returned error %v, want nil", healthcheck, err) } err = verifyNetLBIPv4ForwardingRule(l4NetLB, backendService.SelfLink)