From ee8c3fb5721f9d0f13cce10baf9851db8859d9e2 Mon Sep 17 00:00:00 2001
From: Eoin Gallinagh <eoingallinagh@gmail.com>
Date: Tue, 7 Nov 2023 11:29:56 +0000
Subject: [PATCH] add: caching for Discovery Client
---
.../app/discovery/discovery.go | 39 +++++++++++++++++++
cmd/kar-controllers/app/server.go | 8 ++++
.../genericresource/genericresource.go | 12 +++---
3 files changed, 53 insertions(+), 6 deletions(-)
create mode 100644 cmd/kar-controllers/app/discovery/discovery.go
diff --git a/cmd/kar-controllers/app/discovery/discovery.go b/cmd/kar-controllers/app/discovery/discovery.go
new file mode 100644
index 00000000..b5d26740
--- /dev/null
+++ b/cmd/kar-controllers/app/discovery/discovery.go
@@ -0,0 +1,39 @@
+package discovery
+
+import (
+ "k8s.io/klog/v2"
+ "time"
+
+ "k8s.io/client-go/rest"
+ "k8s.io/client-go/discovery"
+ "k8s.io/client-go/discovery/cached/memory"
+)
+
+var GlobalCachedDiscoveryClient discovery.CachedDiscoveryInterface
+
+func InitializeGlobalDiscoveryClient(config *rest.Config) error {
+ discoveryClient, err := discovery.NewDiscoveryClientForConfig(config)
+ if err != nil {
+ return err
+ }
+ GlobalCachedDiscoveryClient = memory.NewMemCacheClient(discoveryClient)
+
+ go startDiscoveryRefreshTicker()
+
+ return nil
+}
+
+func RefreshDiscoveryCache() {
+ klog.Infof("Invalidating discovery cache")
+ GlobalCachedDiscoveryClient.Invalidate()
+}
+
+func startDiscoveryRefreshTicker() {
+ ticker := time.NewTicker(5 * time.Hour)
+ for {
+ select {
+ case <-ticker.C:
+ RefreshDiscoveryCache()
+ }
+ }
+}
diff --git a/cmd/kar-controllers/app/server.go b/cmd/kar-controllers/app/server.go
index 4deb4ac1..3629f1b3 100644
--- a/cmd/kar-controllers/app/server.go
+++ b/cmd/kar-controllers/app/server.go
@@ -18,6 +18,8 @@ package app
import (
"net/http"
+ "k8s.io/klog/v2"
+ "github.com/project-codeflare/multi-cluster-app-dispatcher/cmd/kar-controllers/app/discovery"
"strings"
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
@@ -62,6 +64,12 @@ func Run(opt *options.ServerOption) error {
AgentConfigs: strings.Split(opt.AgentConfigs, ","),
}
+ if err = discovery.InitializeGlobalDiscoveryClient(restConfig); err != nil {
+ klog.Errorf("Error initializing global discovery client: %s", err)
+ } else {
+ klog.Infof("Initializing global discovery client")
+ }
+
jobctrl := queuejob.NewJobController(restConfig, mcadConfig, extConfig)
if jobctrl == nil {
return nil
diff --git a/pkg/controller/queuejobresources/genericresource/genericresource.go b/pkg/controller/queuejobresources/genericresource/genericresource.go
index a1057f76..11f8e9a3 100644
--- a/pkg/controller/queuejobresources/genericresource/genericresource.go
+++ b/pkg/controller/queuejobresources/genericresource/genericresource.go
@@ -24,7 +24,8 @@ import (
"runtime/debug"
"strings"
"time"
-
+
+ discoveryCache "github.com/project-codeflare/multi-cluster-app-dispatcher/cmd/kar-controllers/app/discovery"
arbv1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1"
v1 "k8s.io/api/core/v1"
"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
@@ -86,8 +87,8 @@ func (gr *GenericResources) Cleanup(aw *arbv1.AppWrapper, awr *arbv1.AppWrapperG
name := ""
namespaced := true
- // todo:DELETEME dd := common.KubeClient.Discovery()
- dd := gr.clients.Discovery()
+
+ dd := discoveryCache.GlobalCachedDiscoveryClient
apigroups, err := restmapper.GetAPIGroupResources(dd)
if err != nil {
klog.Errorf("[Cleanup] Error getting API resources, err=%#v", err)
@@ -206,8 +207,7 @@ func (gr *GenericResources) SyncQueueJob(aw *arbv1.AppWrapper, awr *arbv1.AppWra
}()
namespaced := true
- // todo:DELETEME dd := common.KubeClient.Discovery()
- dd := gr.clients.Discovery()
+ dd := discoveryCache.GlobalCachedDiscoveryClient
apigroups, err := restmapper.GetAPIGroupResources(dd)
if err != nil {
klog.Errorf("Error getting API resources, err=%#v", err)
@@ -624,7 +624,7 @@ func getContainerResources(container v1.Container, replicas float64) *clustersta
// returns status of an item present in etcd
func (gr *GenericResources) IsItemCompleted(awgr *arbv1.AppWrapperGenericResource, namespace string, appwrapperName string, genericItemName string) (completed bool) {
- dd := gr.clients.Discovery()
+ dd := discoveryCache.GlobalCachedDiscoveryClient
apigroups, err := restmapper.GetAPIGroupResources(dd)
if err != nil {
klog.Errorf("[IsItemCompleted] Error getting API resources, err=%#v", err)