Skip to content

Commit 123a085

Browse files
08volt08volt
committed
feature(l4): Add zonal affinity support and related controls
This commit implements zonal affinity capabilities for L4 Internal Load Balancers: - Add zonal affinity flag for L4 ILB controllers - Implement backendService zonal affinity control logic - Add backendService minimum API version control - Update backendService tests to include zonal affinity cases with expected NetworkPassThroughLbTrafficPolicy - Add handler for service spec Spec.TrafficDistribution change events - Add L4 TestZonalAffinity integration tests
1 parent f683eee commit 123a085

File tree

10 files changed

+307
-5
lines changed

10 files changed

+307
-5
lines changed

‎cmd/glbc/main.go

+1
Original file line numberDiff line numberDiff line change
@@ -334,6 +334,7 @@ func main() {
334334
EnableL4NetLBNEGsDefault: flags.F.EnableL4NetLBNEGDefault,
335335
EnableL4ILBMixedProtocol: flags.F.EnableL4ILBMixedProtocol,
336336
EnableL4NetLBMixedProtocol: flags.F.EnableL4NetLBMixedProtocol,
337+
EnableZonalAffinity: flags.F.EnableZonalAffinity,
337338
}
338339
ctx, err := ingctx.NewControllerContext(kubeClient, backendConfigClient, frontendConfigClient, firewallCRClient, svcNegClient, svcAttachmentClient, networkClient, nodeTopologyClient, eventRecorderKubeClient, cloud, namer, kubeSystemUID, ctxConfig, rootLogger)
339340
if err != nil {

‎pkg/backends/backends.go

+94-4
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,9 @@ const (
3535
DefaultConnectionDrainingTimeoutSeconds = 30
3636
defaultTrackingMode = "PER_CONNECTION"
3737
PerSessionTrackingMode = "PER_SESSION" // the only one supported with strong session affinity
38+
DefaultZonalAffinitySpillover = "ZONAL_AFFINITY_SPILL_CROSS_ZONE"
39+
DefaultZonalAffinitySpilloverRatio = 0
40+
ZonalAffinityDisabledSpillover = "ZONAL_AFFINITY_DISABLED"
3841
)
3942

4043
// LocalityLBPolicyType is the type of locality lb policy the backend service should use.
@@ -90,6 +93,31 @@ type L4BackendServiceParams struct {
9093
NetworkInfo *network.NetworkInfo
9194
ConnectionTrackingPolicy *composite.BackendServiceConnectionTrackingPolicy
9295
LocalityLbPolicy LocalityLBPolicyType
96+
EnableZonalAffinity bool
97+
}
98+
99+
var versionPrecedence = map[meta.Version]int{
100+
meta.VersionAlpha: 2,
101+
meta.VersionBeta: 1,
102+
meta.VersionGA: 0,
103+
}
104+
105+
// maxVersion returns the higher version based on precedence.
106+
func maxVersion(a, b meta.Version) meta.Version {
107+
precedenceA, okA := versionPrecedence[a]
108+
if !okA {
109+
precedenceA = 0 // Use VersionGA precedence if invalid.
110+
}
111+
112+
precedenceB, okB := versionPrecedence[b]
113+
if !okB {
114+
precedenceB = 0 // Use VersionGA precedence if invalid.
115+
}
116+
117+
if precedenceA > precedenceB {
118+
return a
119+
}
120+
return b
93121
}
94122

95123
// ensureDescription updates the BackendService Description with the expected value
@@ -327,6 +355,14 @@ func (p *Pool) DeleteSignedURLKey(be *composite.BackendService, keyName string,
327355
return nil
328356
}
329357

358+
// minRequiredVersion to create a backend service with the given params
359+
func minRequiredVersion(params L4BackendServiceParams) meta.Version {
360+
if params.EnableZonalAffinity {
361+
return meta.VersionAlpha
362+
}
363+
return meta.VersionGA
364+
}
365+
330366
// EnsureL4BackendService creates or updates the backend service with the given name.
331367
func (p *Pool) EnsureL4BackendService(params L4BackendServiceParams, beLogger klog.Logger) (*composite.BackendService, utils.ResourceSyncStatus, error) {
332368
start := time.Now()
@@ -342,25 +378,34 @@ func (p *Pool) EnsureL4BackendService(params L4BackendServiceParams, beLogger kl
342378
if err != nil {
343379
return nil, utils.ResourceResync, err
344380
}
381+
345382
currentBS, err := composite.GetBackendService(p.cloud, key, meta.VersionGA, beLogger)
346383
if err != nil && !utils.IsNotFoundError(err) {
347384
return nil, utils.ResourceResync, err
348385
}
349-
desc, err := utils.MakeL4LBServiceDescription(params.NamespacedName.String(), "", meta.VersionGA, false, utils.ILB)
386+
387+
expectedVersion := minRequiredVersion(params)
388+
expectedDesc, err := utils.MakeL4LBServiceDescription(params.NamespacedName.String(), "", expectedVersion, false, utils.ILB)
350389
if err != nil {
351390
beLogger.Info("EnsureL4BackendService: Failed to generate description for BackendService", "err", err)
352391
}
353392

354393
expectedBS := &composite.BackendService{
355394
Name: params.Name,
356395
Protocol: params.Protocol,
357-
Description: desc,
396+
Version: expectedVersion,
397+
Description: expectedDesc,
358398
HealthChecks: []string{params.HealthCheckLink},
359399
SessionAffinity: utils.TranslateAffinityType(params.SessionAffinity, beLogger),
360400
LoadBalancingScheme: params.Scheme,
361401
LocalityLbPolicy: string(params.LocalityLbPolicy),
362402
}
363403

404+
if params.EnableZonalAffinity {
405+
beLogger.V(2).Info("EnsureL4BackendService: using Zonal Affinity", "spillover", DefaultZonalAffinitySpillover, "spilloverRatio", DefaultZonalAffinitySpilloverRatio)
406+
expectedBS.NetworkPassThroughLbTrafficPolicy = defaultZonalAffinityTrafficPolicy()
407+
}
408+
364409
// We need this configuration only for Strong Session Affinity feature
365410
if p.useConnectionTrackingPolicy {
366411
beLogger.V(2).Info(fmt.Sprintf("EnsureL4BackendService: using connection tracking policy: %+v", params.ConnectionTrackingPolicy))
@@ -388,7 +433,7 @@ func (p *Pool) EnsureL4BackendService(params L4BackendServiceParams, beLogger kl
388433
// We need to perform a GCE call to re-fetch the object we just created
389434
// so that the "Fingerprint" field is filled in. This is needed to update the
390435
// object without error. The lookup is also needed to populate the selfLink.
391-
createdBS, err := composite.GetBackendService(p.cloud, key, meta.VersionGA, beLogger)
436+
createdBS, err := composite.GetBackendService(p.cloud, key, expectedBS.Version, beLogger)
392437
return createdBS, utils.ResourceUpdate, err
393438
} else {
394439
// TODO(FelipeYepez) remove this check once LocalityLBPolicyMaglev does not require allow lisiting
@@ -398,6 +443,17 @@ func (p *Pool) EnsureL4BackendService(params L4BackendServiceParams, beLogger kl
398443

399444
expectedBS.LocalityLbPolicy = string(LocalityLBPolicyMaglev)
400445
}
446+
447+
// Use the version with most priority if the BackendService was already using one
448+
currentVersion := meta.VersionGA
449+
var currentDesc utils.L4LBResourceDescription
450+
err = currentDesc.Unmarshal(currentBS.Description)
451+
if err != nil {
452+
beLogger.V(0).Error(err, "EnsureL4BackendService: error unmarshaling backend service description")
453+
} else {
454+
currentVersion = currentDesc.APIVersion
455+
}
456+
expectedBS.Version = maxVersion(currentVersion, expectedBS.Version)
401457
}
402458

403459
if backendSvcEqual(expectedBS, currentBS, p.useConnectionTrackingPolicy) {
@@ -418,7 +474,7 @@ func (p *Pool) EnsureL4BackendService(params L4BackendServiceParams, beLogger kl
418474
}
419475
beLogger.V(2).Info("EnsureL4BackendService: updated backend service successfully")
420476

421-
updatedBS, err := composite.GetBackendService(p.cloud, key, meta.VersionGA, beLogger)
477+
updatedBS, err := composite.GetBackendService(p.cloud, key, expectedBS.Version, beLogger)
422478
return updatedBS, utils.ResourceUpdate, err
423479
}
424480

@@ -447,9 +503,34 @@ func backendSvcEqual(newBS, oldBS *composite.BackendService, compareConnectionTr
447503
(newBS.LocalityLbPolicy == string(LocalityLBPolicyDefault) && oldBS.LocalityLbPolicy == string(LocalityLBPolicyMaglev)) ||
448504
(newBS.LocalityLbPolicy == string(LocalityLBPolicyMaglev) && oldBS.LocalityLbPolicy == string(LocalityLBPolicyDefault)))
449505

506+
// If zonal affinity is set, needs to be equal
507+
svcsEqual = svcsEqual && zonalAffinityEqual(newBS, oldBS)
450508
return svcsEqual
451509
}
452510

511+
func convertNetworkLbTrafficPolicyToZonalAffinity(trafficPolicy *composite.BackendServiceNetworkPassThroughLbTrafficPolicy) composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity {
512+
if trafficPolicy == nil || trafficPolicy.ZonalAffinity == nil {
513+
return composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{
514+
Spillover: ZonalAffinityDisabledSpillover,
515+
SpilloverRatio: 0,
516+
}
517+
}
518+
519+
return *trafficPolicy.ZonalAffinity
520+
}
521+
522+
func zonalAffinityEqual(a, b *composite.BackendService) bool {
523+
aZonalAffinity := convertNetworkLbTrafficPolicyToZonalAffinity(a.NetworkPassThroughLbTrafficPolicy)
524+
bZonalAffinity := convertNetworkLbTrafficPolicyToZonalAffinity(b.NetworkPassThroughLbTrafficPolicy)
525+
526+
// Compare Spillover values
527+
spilloverEqual := aZonalAffinity.Spillover == bZonalAffinity.Spillover
528+
// Compare SpilloverRatio values
529+
spilloverRatioEqual := aZonalAffinity.SpilloverRatio == bZonalAffinity.SpilloverRatio
530+
531+
return spilloverEqual && spilloverRatioEqual
532+
}
533+
453534
// connectionTrackingPolicyEqual returns true if both elements are equal
454535
// and return false if at least one parameter is different
455536
func connectionTrackingPolicyEqual(a, b *composite.BackendServiceConnectionTrackingPolicy) bool {
@@ -460,3 +541,12 @@ func connectionTrackingPolicyEqual(a, b *composite.BackendServiceConnectionTrack
460541
a.EnableStrongAffinity == b.EnableStrongAffinity &&
461542
a.IdleTimeoutSec == b.IdleTimeoutSec
462543
}
544+
545+
func defaultZonalAffinityTrafficPolicy() *composite.BackendServiceNetworkPassThroughLbTrafficPolicy {
546+
return &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{
547+
ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{
548+
Spillover: DefaultZonalAffinitySpillover,
549+
SpilloverRatio: DefaultZonalAffinitySpilloverRatio,
550+
},
551+
}
552+
}

‎pkg/backends/backends_test.go

+86
Original file line numberDiff line numberDiff line change
@@ -626,6 +626,92 @@ func TestBackendSvcEqual(t *testing.T) {
626626
},
627627
wantEqual: false,
628628
},
629+
{
630+
desc: "Test existing backend service diff with zonal affinity feature enabled",
631+
oldBackendService: &composite.BackendService{
632+
NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{
633+
ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{
634+
Spillover: "ZONAL_AFFINITY_SPILL_CROSS_ZONE",
635+
SpilloverRatio: 0.7,
636+
},
637+
},
638+
},
639+
newBackendService: &composite.BackendService{
640+
NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{
641+
ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{
642+
Spillover: "ZONAL_AFFINITY_SPILL_CROSS_ZONE",
643+
SpilloverRatio: 0.7,
644+
},
645+
},
646+
},
647+
wantEqual: true,
648+
},
649+
{
650+
desc: "Test existing backend service diff with zonal affinity feature enabled but different ratio",
651+
oldBackendService: &composite.BackendService{
652+
NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{
653+
ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{
654+
Spillover: "ZONAL_AFFINITY_SPILL_CROSS_ZONE",
655+
SpilloverRatio: 0.7,
656+
},
657+
},
658+
},
659+
newBackendService: &composite.BackendService{
660+
NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{
661+
ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{
662+
Spillover: "ZONAL_AFFINITY_SPILL_CROSS_ZONE",
663+
SpilloverRatio: 0.3,
664+
},
665+
},
666+
},
667+
wantEqual: false,
668+
},
669+
{
670+
desc: "Test existing backend service diff with zonal affinity feature enabled but different spillover strategy",
671+
oldBackendService: &composite.BackendService{
672+
NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{
673+
ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{
674+
Spillover: "ZONAL_AFFINITY_STAY_WITHIN_ZONE",
675+
SpilloverRatio: 0.3,
676+
},
677+
},
678+
},
679+
newBackendService: &composite.BackendService{
680+
NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{
681+
ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{
682+
Spillover: "ZONAL_AFFINITY_SPILL_CROSS_ZONE",
683+
SpilloverRatio: 0.3,
684+
},
685+
},
686+
},
687+
wantEqual: false,
688+
},
689+
{
690+
desc: "Test existing backend service diff enabling zonal affinity feature",
691+
oldBackendService: &composite.BackendService{},
692+
newBackendService: &composite.BackendService{
693+
NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{
694+
ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{
695+
Spillover: "ZONAL_AFFINITY_SPILL_CROSS_ZONE",
696+
SpilloverRatio: 0.3,
697+
},
698+
},
699+
},
700+
wantEqual: false,
701+
},
702+
{
703+
desc: "Test existing backend service diff enabling zonal affinity feature",
704+
oldBackendService: &composite.BackendService{
705+
NetworkPassThroughLbTrafficPolicy: &composite.BackendServiceNetworkPassThroughLbTrafficPolicy{
706+
ZonalAffinity: &composite.BackendServiceNetworkPassThroughLbTrafficPolicyZonalAffinity{
707+
Spillover: "ZONAL_AFFINITY_SPILL_CROSS_ZONE",
708+
SpilloverRatio: 0.3,
709+
},
710+
},
711+
},
712+
newBackendService: &composite.BackendService{},
713+
wantEqual: false,
714+
},
629715
} {
630716
tc := tc
631717
t.Run(tc.desc, func(t *testing.T) {

‎pkg/context/context.go

+1
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,7 @@ type ControllerContextConfig struct {
133133
EnableIngressRegionalExternal bool
134134
EnableWeightedL4ILB bool
135135
EnableWeightedL4NetLB bool
136+
EnableZonalAffinity bool
136137
DisableL4LBFirewall bool
137138
EnableL4NetLBNEGs bool
138139
EnableL4NetLBNEGsDefault bool

‎pkg/flags/flags.go

+2
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@ var F = struct {
139139
EnableWeightedL4NetLB bool
140140
EnableDiscretePortForwarding bool
141141
EnableMultiProjectMode bool
142+
EnableZonalAffinity bool
142143
ProviderConfigNameLabelKey string
143144
EnableL4ILBMixedProtocol bool
144145
EnableL4NetLBMixedProtocol bool
@@ -326,6 +327,7 @@ L7 load balancing. CSV values accepted. Example: -node-port-ranges=80,8080,400-5
326327
flag.StringVar(&F.NodeTopologyCRName, "node-topology-cr-name", "default", "The name of the Node Topology CR.")
327328
flag.BoolVar(&F.EnableWeightedL4ILB, "enable-weighted-l4-ilb", false, "Enable Weighted Load balancing for L4 ILB.")
328329
flag.BoolVar(&F.EnableWeightedL4NetLB, "enable-weighted-l4-netlb", false, "EnableWeighted Load balancing for L4 NetLB .")
330+
flag.BoolVar(&F.EnableZonalAffinity, "enable-zonal-affinity", false, "Enable Zonal Affinity for L4 ILB.")
329331
flag.Float32Var(&F.KubeClientQPS, "kube-client-qps", 0.0, "The QPS that the controllers' kube client should adhere to through client side throttling. If zero, client will be created with default settings.")
330332
flag.IntVar(&F.KubeClientBurst, "kube-client-burst", 0, "The burst QPS that the controllers' kube client should adhere to through client side throttling. If zero, client will be created with default settings.")
331333
flag.BoolVar(&F.EnableDiscretePortForwarding, "enable-discrete-port-forwarding", false, "Enable forwarding of individual ports instead of port ranges.")

‎pkg/l4lb/l4controller.go

+7
Original file line numberDiff line numberDiff line change
@@ -287,6 +287,7 @@ func (l4c *L4Controller) processServiceCreateOrUpdate(service *v1.Service, svcLo
287287
EnableWeightedLB: l4c.ctx.EnableWeightedL4ILB,
288288
DisableNodesFirewallProvisioning: l4c.ctx.DisableL4LBFirewall,
289289
EnableMixedProtocol: l4c.ctx.EnableL4ILBMixedProtocol,
290+
EnableZonalAffinity: l4c.ctx.EnableZonalAffinity,
290291
}
291292
l4 := loadbalancers.NewL4Handler(l4ilbParams, svcLogger)
292293
syncResult := l4.EnsureInternalLoadBalancer(utils.GetNodeNames(nodes), service)
@@ -369,6 +370,7 @@ func (l4c *L4Controller) processServiceDeletion(key string, svc *v1.Service, svc
369370
EnableWeightedLB: l4c.ctx.EnableWeightedL4ILB,
370371
DisableNodesFirewallProvisioning: l4c.ctx.DisableL4LBFirewall,
371372
EnableMixedProtocol: l4c.ctx.EnableL4ILBMixedProtocol,
373+
EnableZonalAffinity: l4c.ctx.EnableZonalAffinity,
372374
}
373375
l4 := loadbalancers.NewL4Handler(l4ilbParams, svcLogger)
374376
l4c.ctx.Recorder(svc.Namespace).Eventf(svc, v1.EventTypeNormal, "DeletingLoadBalancer", "Deleting load balancer for %s", key)
@@ -579,6 +581,11 @@ func (l4c *L4Controller) needsUpdate(oldService *v1.Service, newService *v1.Serv
579581
oldService.Spec.HealthCheckNodePort, newService.Spec.HealthCheckNodePort)
580582
return true
581583
}
584+
if oldService.Spec.TrafficDistribution != newService.Spec.TrafficDistribution {
585+
recorder.Eventf(newService, v1.EventTypeNormal, "TrafficDistribution", "%v -> %v",
586+
oldService.Spec.TrafficDistribution, newService.Spec.TrafficDistribution)
587+
return true
588+
}
582589
if l4c.enableDualStack && !reflect.DeepEqual(oldService.Spec.IPFamilies, newService.Spec.IPFamilies) {
583590
recorder.Eventf(newService, v1.EventTypeNormal, "IPFamilies", "%v -> %v",
584591
oldService.Spec.IPFamilies, newService.Spec.IPFamilies)

‎pkg/l4lb/l4netlbcontroller_test.go

+1
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ const (
7272
usersIP = "35.10.211.60"
7373
testServiceNamespace = "default"
7474
hcNodePort = int32(10111)
75+
trafficDistribution = "PreferClose"
7576
userAddrName = "UserStaticAddress"
7677

7778
shortSessionAffinityIdleTimeout = int32(20) // 20 sec could be used for regular Session Affinity

0 commit comments

Comments
 (0)