From 505ab0c79e94acdd18453771a72b055821f24d4a Mon Sep 17 00:00:00 2001 From: Karol Wychowaniec Date: Thu, 31 Aug 2017 17:35:17 +0200 Subject: [PATCH 1/3] Create e2e test for Custom Metrics - Stackdriver Adapter --- test/e2e/instrumentation/monitoring/BUILD | 9 + .../monitoring/custom_metrics_deployments.go | 91 ++++++++ .../monitoring/custom_metrics_stackdriver.go | 200 ++++++++++++++++++ .../instrumentation/monitoring/stackdriver.go | 14 ++ 4 files changed, 314 insertions(+) create mode 100644 test/e2e/instrumentation/monitoring/custom_metrics_deployments.go create mode 100644 test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go diff --git a/test/e2e/instrumentation/monitoring/BUILD b/test/e2e/instrumentation/monitoring/BUILD index 7677ba55d7cc1..9bb131728f456 100644 --- a/test/e2e/instrumentation/monitoring/BUILD +++ b/test/e2e/instrumentation/monitoring/BUILD @@ -9,6 +9,8 @@ go_library( name = "go_default_library", srcs = [ "cadvisor.go", + "custom_metrics_deployments.go", + "custom_metrics_stackdriver.go", "influxdb.go", "metrics_grabber.go", "stackdriver.go", @@ -21,12 +23,19 @@ go_library( "//vendor/github.com/influxdata/influxdb/client/v2:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", + "//vendor/golang.org/x/oauth2:go_default_library", "//vendor/golang.org/x/oauth2/google:go_default_library", "//vendor/google.golang.org/api/monitoring/v3:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/selection:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//vendor/k8s.io/client-go/discovery:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", + "//vendor/k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset:go_default_library", + "//vendor/k8s.io/metrics/pkg/client/custom_metrics:go_default_library", ], ) diff --git a/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go b/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go new file mode 100644 index 0000000000000..910dbe9df7f0e --- /dev/null +++ b/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go @@ -0,0 +1,91 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package monitoring + +import ( + "fmt" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var ( + CustomMetricName = "foo-metric" + UnusedMetricName = "unused-metric" + MetricValue1 = int64(448) + MetricValue2 = int64(446) + + SDExporterPod1 = &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sd-exporter-1", + Namespace: "default", + Labels: map[string]string{ + "name": "sd-exporter", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "sd-exporter", + Image: "gcr.io/google-containers/sd-dummy-exporter:v0.1.0", + ImagePullPolicy: v1.PullPolicy("Always"), + Command: []string{"/sd_dummy_exporter", "--pod-id=$(POD_ID)", "--metric-name=" + CustomMetricName, fmt.Sprintf("--metric-value=%v", MetricValue1)}, + Env: []v1.EnvVar{ + { + Name: "POD_ID", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "metadata.uid", + }, + }, + }, + }, + Ports: []v1.ContainerPort{{ContainerPort: 80}}, + }, + }, + }, + } + SDExporterPod2 = &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sd-exporter-2", + Namespace: "default", + Labels: map[string]string{ + "name": "sd-exporter", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "sd-exporter", + Image: "gcr.io/google-containers/sd-dummy-exporter:v0.1.0", + ImagePullPolicy: v1.PullPolicy("Always"), + Command: []string{"/sd_dummy_exporter", "--pod-id=$(POD_ID)", "--metric-name=" + CustomMetricName, fmt.Sprintf("--metric-value=%v", MetricValue2)}, + Env: []v1.EnvVar{ + { + Name: "POD_ID", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "metadata.uid", + }, + }, + }, + }, + Ports: []v1.ContainerPort{{ContainerPort: 80}}, + }, + }, + }, + } +) diff --git a/test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go b/test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go new file mode 100644 index 0000000000000..3aaa0ab27e623 --- /dev/null +++ b/test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go @@ -0,0 +1,200 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package monitoring + +import ( + "context" + "fmt" + "time" + + "golang.org/x/oauth2/google" + clientset "k8s.io/client-go/kubernetes" + + . "github.com/onsi/ginkgo" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/test/e2e/framework" + instrumentation "k8s.io/kubernetes/test/e2e/instrumentation/common" + + gcm "google.golang.org/api/monitoring/v3" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/client-go/discovery" + kubeaggrcs "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" + customclient "k8s.io/metrics/pkg/client/custom_metrics" +) + +var _ = instrumentation.SIGDescribe("Stackdriver Monitoring", func() { + BeforeEach(func() { + framework.SkipUnlessProviderIs("gce", "gke") + }) + + f := framework.NewDefaultFramework("stackdriver-monitoring") + var kubeClient clientset.Interface + var kubeAggrClient kubeaggrcs.Interface + var customMetricsClient customclient.CustomMetricsClient + var discoveryClient *discovery.DiscoveryClient + + It("should run Custom Metrics - Stackdriver Adapter [Feature:StackdriverMonitoring]", func() { + kubeClient = f.ClientSet + kubeAggrClient = f.AggregatorClient + config, err := framework.LoadConfig() + if err != nil { + framework.Failf("Failed to load config: %s", err) + } + customMetricsClient = customclient.NewForConfigOrDie(config) + discoveryClient = discovery.NewDiscoveryClientForConfigOrDie(config) + testAdapter(f, kubeClient, kubeAggrClient, customMetricsClient, discoveryClient) + }) +}) + +func testAdapter(f *framework.Framework, kubeClient clientset.Interface, kubeAggrClient kubeaggrcs.Interface, customMetricsClient customclient.CustomMetricsClient, discoveryClient *discovery.DiscoveryClient) { + projectId := framework.TestContext.CloudConfig.ProjectID + + ctx := context.Background() + client, err := google.DefaultClient(ctx, gcm.CloudPlatformScope) + + // Hack for running tests locally + // If this is your use case, create application default credentials: + // $ gcloud auth application-default login + // and uncomment following lines (comment out the two lines above): + /* + ts, err := google.DefaultTokenSource(oauth2.NoContext) + framework.Logf("Couldn't get application default credentials, %v", err) + if err != nil { + framework.Failf("Error accessing application default credentials, %v", err) + } + client := oauth2.NewClient(oauth2.NoContext, ts) + */ + + gcmService, err := gcm.New(client) + if err != nil { + framework.Failf("Failed to create gcm service, %v", err) + } + + framework.ExpectNoError(err) + + // Set up a cluster: create a custom metric and set up k8s-sd adapter + err = createDescriptors(gcmService, projectId) + if err != nil { + framework.Failf("Failed to create metric descriptor: %s", err) + } + defer cleanupDescriptors(gcmService, projectId) + + err = createAdapter() + if err != nil { + framework.Failf("Failed to set up: %s", err) + } + defer cleanupAdapter() + + // Run application that exports the metric + err = createSDExporterPod(kubeClient) + if err != nil { + framework.Failf("Failed to create sd-exporter pod: %s", err) + } + defer cleanupSDExporterPod(kubeClient) + + // Wait a short amount of time to create a pod and export some metrics + time.Sleep(60 * time.Second) + + // Verify responses from Custom Metrics API + resources, err := discoveryClient.ServerResourcesForGroupVersion("custom.metrics.k8s.io/v1beta1") + if err != nil { + framework.Failf("Failed to retrieve a list of supported metrics: %s", err) + } + for _, resource := range resources.APIResources { + if resource.Name != "pods/"+CustomMetricName && resource.Name != "pods/"+UnusedMetricName { + framework.Failf("Unexpected metric %s. Only metric %s should be supported", resource.Name, CustomMetricName) + } + } + value, err := customMetricsClient.NamespacedMetrics("default").GetForObject(schema.GroupKind{Group: "", Kind: "Pod"}, "sd-exporter-1", CustomMetricName) + if err != nil { + framework.Failf("Failed query: %s", err) + } + if value.Value.Value() != MetricValue1 { + framework.Failf("Unexpected metric value for metric %s: expected %v but received %v", CustomMetricName, MetricValue1, value.Value) + } + filter, err := labels.NewRequirement("name", selection.Equals, []string{"sd-exporter"}) + if err != nil { + framework.Failf("Couldn't create a label filter") + } + values, err := customMetricsClient.NamespacedMetrics("default").GetForObjects(schema.GroupKind{Group: "", Kind: "Pod"}, labels.NewSelector().Add(*filter), CustomMetricName) + if err != nil { + framework.Failf("Failed query: %s", err) + } + if len(values.Items) != 2 { + framework.Failf("Expected results for exactly 2 pods, but %v results received", len(values.Items)) + } + for _, value := range values.Items { + if (value.DescribedObject.Name == SDExporterPod1.Name && value.Value.Value() != MetricValue1) || + (value.DescribedObject.Name == SDExporterPod2.Name && value.Value.Value() != MetricValue2) { + framework.Failf("Unexpected metric value for metric %s and pod %s: %v", CustomMetricName, value.DescribedObject.Name, value.Value.Value()) + } + } + + framework.ExpectNoError(err) +} + +func createDescriptors(service *gcm.Service, projectId string) error { + _, err := service.Projects.MetricDescriptors.Create(fmt.Sprintf("projects/%s", projectId), &gcm.MetricDescriptor{ + Name: CustomMetricName, + ValueType: "INT64", + Type: "custom.googleapis.com/" + CustomMetricName, + MetricKind: "GAUGE", + }).Do() + if err != nil { + return err + } + _, err = service.Projects.MetricDescriptors.Create(fmt.Sprintf("projects/%s", projectId), &gcm.MetricDescriptor{ + Name: UnusedMetricName, + ValueType: "INT64", + Type: "custom.googleapis.com/" + UnusedMetricName, + MetricKind: "GAUGE", + }).Do() + return err +} + +func createSDExporterPod(cs clientset.Interface) error { + _, err := cs.Core().Pods("default").Create(SDExporterPod1) + if err != nil { + return err + } + _, err = cs.Core().Pods("default").Create(SDExporterPod2) + return err +} + +func createAdapter() error { + stat, err := framework.RunKubectl("create", "-f", "https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/adapter-beta.yaml") + framework.Logf(stat) + return err +} + +func cleanupDescriptors(service *gcm.Service, projectId string) { + _, _ = service.Projects.MetricDescriptors.Delete(fmt.Sprintf("projects/%s/metricDescriptors/custom.googleapis.com/%s", projectId, CustomMetricName)).Do() + _, _ = service.Projects.MetricDescriptors.Delete(fmt.Sprintf("projects/%s/metricDescriptors/custom.googleapis.com/%s", projectId, UnusedMetricName)).Do() +} + +func cleanupSDExporterPod(cs clientset.Interface) { + _ = cs.Core().Pods("default").Delete("sd-exporter-1", &metav1.DeleteOptions{}) + _ = cs.Core().Pods("default").Delete("sd-exporter-2", &metav1.DeleteOptions{}) +} + +func cleanupAdapter() error { + stat, err := framework.RunKubectl("delete", "-f", "https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/adapter-beta.yaml") + framework.Logf(stat) + return err +} diff --git a/test/e2e/instrumentation/monitoring/stackdriver.go b/test/e2e/instrumentation/monitoring/stackdriver.go index 3028fdbe5ec7c..6923ce2e1fcd9 100644 --- a/test/e2e/instrumentation/monitoring/stackdriver.go +++ b/test/e2e/instrumentation/monitoring/stackdriver.go @@ -76,6 +76,20 @@ func testStackdriverMonitoring(f *framework.Framework, pods, allPodsCPU int, per ctx := context.Background() client, err := google.DefaultClient(ctx, gcm.CloudPlatformScope) + + // Hack for running tests locally + // If this is your use case, create application default credentials: + // $ gcloud auth application-default login + // and uncomment following lines (comment out the two lines above): (DON'T set the env var below) + /* + ts, err := google.DefaultTokenSource(oauth2.NoContext) + framework.Logf("Couldn't get application default credentials, %v", err) + if err != nil { + framework.Failf("Error accessing application default credentials, %v", err) + } + client := oauth2.NewClient(oauth2.NoContext, ts) + */ + gcmService, err := gcm.New(client) // set this env var if accessing Stackdriver test endpoint (default is prod): From 5bad51f6162a98f80fb83099d0552cca0b26c189 Mon Sep 17 00:00:00 2001 From: Karol Wychowaniec Date: Mon, 25 Sep 2017 17:01:46 +0200 Subject: [PATCH 2/3] Add test for HPA --- test/e2e/autoscaling/BUILD | 5 + .../autoscaling/custom_metrics_autoscaling.go | 210 ++++++++++++++++++ test/e2e/instrumentation/monitoring/BUILD | 2 +- .../monitoring/custom_metrics_deployments.go | 149 +++++++++---- .../monitoring/custom_metrics_stackdriver.go | 113 ++++------ .../instrumentation/monitoring/stackdriver.go | 12 +- 6 files changed, 366 insertions(+), 125 deletions(-) create mode 100644 test/e2e/autoscaling/custom_metrics_autoscaling.go diff --git a/test/e2e/autoscaling/BUILD b/test/e2e/autoscaling/BUILD index a22b273c4a6c0..43ea4c8725b4d 100644 --- a/test/e2e/autoscaling/BUILD +++ b/test/e2e/autoscaling/BUILD @@ -11,6 +11,7 @@ go_library( "autoscaling_timer.go", "cluster_autoscaler_scalability.go", "cluster_size_autoscaling.go", + "custom_metrics_autoscaling.go", "dns_autoscaling.go", "framework.go", "horizontal_pod_autoscaling.go", @@ -19,11 +20,15 @@ go_library( "//pkg/api:go_default_library", "//test/e2e/common:go_default_library", "//test/e2e/framework:go_default_library", + "//test/e2e/instrumentation/monitoring:go_default_library", "//test/e2e/scheduling:go_default_library", "//test/utils:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", + "//vendor/golang.org/x/oauth2/google:go_default_library", + "//vendor/google.golang.org/api/monitoring/v3:go_default_library", + "//vendor/k8s.io/api/autoscaling/v2beta1:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/api/policy/v1beta1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", diff --git a/test/e2e/autoscaling/custom_metrics_autoscaling.go b/test/e2e/autoscaling/custom_metrics_autoscaling.go new file mode 100644 index 0000000000000..7a633842bcaee --- /dev/null +++ b/test/e2e/autoscaling/custom_metrics_autoscaling.go @@ -0,0 +1,210 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package autoscaling + +import ( + "context" + "time" + + "golang.org/x/oauth2/google" + clientset "k8s.io/client-go/kubernetes" + + . "github.com/onsi/ginkgo" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/test/e2e/framework" + + gcm "google.golang.org/api/monitoring/v3" + as "k8s.io/api/autoscaling/v2beta1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubernetes/test/e2e/instrumentation/monitoring" +) + +const ( + stackdriverExporterDeployment = "stackdriver-exporter-deployment" + dummyDeploymentName = "dummy-deployment" + stackdriverExporterPod = "stackdriver-exporter-pod" +) + +var _ = SIGDescribe("[HPA] Horizontal pod autoscaling (scale resource: Custom Metrics from Stackdriver)", func() { + BeforeEach(func() { + framework.SkipUnlessProviderIs("gce") + }) + + f := framework.NewDefaultFramework("horizontal-pod-autoscaling") + var kubeClient clientset.Interface + + It("should autoscale with Custom Metrics from Stackdriver [Feature:CustomMetricsAutoscaling]", func() { + kubeClient = f.ClientSet + testHPA(f, kubeClient) + }) +}) + +func testHPA(f *framework.Framework, kubeClient clientset.Interface) { + projectId := framework.TestContext.CloudConfig.ProjectID + + ctx := context.Background() + client, err := google.DefaultClient(ctx, gcm.CloudPlatformScope) + + // Hack for running tests locally, needed to authenticate in Stackdriver + // If this is your use case, create application default credentials: + // $ gcloud auth application-default login + // and uncomment following lines: + /* + ts, err := google.DefaultTokenSource(oauth2.NoContext) + framework.Logf("Couldn't get application default credentials, %v", err) + if err != nil { + framework.Failf("Error accessing application default credentials, %v", err) + } + client := oauth2.NewClient(oauth2.NoContext, ts) + */ + + gcmService, err := gcm.New(client) + if err != nil { + framework.Failf("Failed to create gcm service, %v", err) + } + + // Set up a cluster: create a custom metric and set up k8s-sd adapter + err = monitoring.CreateDescriptors(gcmService, projectId) + if err != nil { + framework.Failf("Failed to create metric descriptor: %v", err) + } + defer monitoring.CleanupDescriptors(gcmService, projectId) + + err = monitoring.CreateAdapter() + if err != nil { + framework.Failf("Failed to set up: %v", err) + } + defer monitoring.CleanupAdapter() + + // Run application that exports the metric + err = createDeploymentsToScale(f, kubeClient) + if err != nil { + framework.Failf("Failed to create stackdriver-exporter pod: %v", err) + } + defer cleanupDeploymentsToScale(f, kubeClient) + + // Autoscale the deployments + err = createPodsHPA(f, kubeClient) + if err != nil { + framework.Failf("Failed to create 'Pods' HPA: %v", err) + } + err = createObjectHPA(f, kubeClient) + if err != nil { + framework.Failf("Failed to create 'Objects' HPA: %v", err) + } + + waitForReplicas(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, kubeClient, 15*time.Minute, 1) + waitForReplicas(dummyDeploymentName, f.Namespace.ObjectMeta.Name, kubeClient, 15*time.Minute, 1) +} + +func createDeploymentsToScale(f *framework.Framework, cs clientset.Interface) error { + _, err := cs.Extensions().Deployments(f.Namespace.ObjectMeta.Name).Create(monitoring.StackdriverExporterDeployment(stackdriverExporterDeployment, f.Namespace.Name, 2, 100)) + if err != nil { + return err + } + _, err = cs.Core().Pods(f.Namespace.ObjectMeta.Name).Create(monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.Name, stackdriverExporterPod, monitoring.CustomMetricName, 100)) + if err != nil { + return err + } + _, err = cs.Extensions().Deployments(f.Namespace.ObjectMeta.Name).Create(monitoring.StackdriverExporterDeployment(dummyDeploymentName, f.Namespace.Name, 2, 100)) + return err +} + +func cleanupDeploymentsToScale(f *framework.Framework, cs clientset.Interface) { + _ = cs.Extensions().Deployments(f.Namespace.ObjectMeta.Name).Delete(stackdriverExporterDeployment, &metav1.DeleteOptions{}) + _ = cs.Core().Pods(f.Namespace.ObjectMeta.Name).Delete(stackdriverExporterPod, &metav1.DeleteOptions{}) + _ = cs.Extensions().Deployments(f.Namespace.ObjectMeta.Name).Delete(dummyDeploymentName, &metav1.DeleteOptions{}) +} + +func createPodsHPA(f *framework.Framework, cs clientset.Interface) error { + var minReplicas int32 = 1 + _, err := cs.AutoscalingV2beta1().HorizontalPodAutoscalers(f.Namespace.ObjectMeta.Name).Create(&as.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "custom-metrics-pods-hpa", + Namespace: f.Namespace.ObjectMeta.Name, + }, + Spec: as.HorizontalPodAutoscalerSpec{ + Metrics: []as.MetricSpec{ + { + Type: as.PodsMetricSourceType, + Pods: &as.PodsMetricSource{ + MetricName: monitoring.CustomMetricName, + TargetAverageValue: *resource.NewQuantity(200, resource.DecimalSI), + }, + }, + }, + MaxReplicas: 3, + MinReplicas: &minReplicas, + ScaleTargetRef: as.CrossVersionObjectReference{ + APIVersion: "extensions/v1beta1", + Kind: "Deployment", + Name: stackdriverExporterDeployment, + }, + }, + }) + return err +} + +func createObjectHPA(f *framework.Framework, cs clientset.Interface) error { + var minReplicas int32 = 1 + _, err := cs.AutoscalingV2beta1().HorizontalPodAutoscalers(f.Namespace.ObjectMeta.Name).Create(&as.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "custom-metrics-objects-hpa", + Namespace: f.Namespace.ObjectMeta.Name, + }, + Spec: as.HorizontalPodAutoscalerSpec{ + Metrics: []as.MetricSpec{ + { + Type: as.ObjectMetricSourceType, + Object: &as.ObjectMetricSource{ + MetricName: monitoring.CustomMetricName, + Target: as.CrossVersionObjectReference{ + Kind: "Pod", + Name: stackdriverExporterPod, + }, + TargetValue: *resource.NewQuantity(200, resource.DecimalSI), + }, + }, + }, + MaxReplicas: 3, + MinReplicas: &minReplicas, + ScaleTargetRef: as.CrossVersionObjectReference{ + APIVersion: "extensions/v1beta1", + Kind: "Deployment", + Name: dummyDeploymentName, + }, + }, + }) + return err +} + +func waitForReplicas(deploymentName, namespace string, cs clientset.Interface, timeout time.Duration, desiredReplicas int) { + interval := 20 * time.Second + err := wait.PollImmediate(interval, timeout, func() (bool, error) { + deployment, err := cs.Extensions().Deployments(namespace).Get(deploymentName, metav1.GetOptions{}) + if err != nil { + framework.Failf("Failed to get replication controller %s: %v", deployment, err) + } + replicas := int(deployment.Status.ReadyReplicas) + framework.Logf("waiting for %d replicas (current: %d)", desiredReplicas, replicas) + return replicas == desiredReplicas, nil // Expected number of replicas found. Exit. + }) + if err != nil { + framework.Failf("Timeout waiting %v for %v replicas", timeout, desiredReplicas) + } +} diff --git a/test/e2e/instrumentation/monitoring/BUILD b/test/e2e/instrumentation/monitoring/BUILD index 9bb131728f456..7d03cdeba6e60 100644 --- a/test/e2e/instrumentation/monitoring/BUILD +++ b/test/e2e/instrumentation/monitoring/BUILD @@ -23,10 +23,10 @@ go_library( "//vendor/github.com/influxdata/influxdb/client/v2:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", - "//vendor/golang.org/x/oauth2:go_default_library", "//vendor/golang.org/x/oauth2/google:go_default_library", "//vendor/google.golang.org/api/monitoring/v3:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/api/extensions/v1beta1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", diff --git a/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go b/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go index 910dbe9df7f0e..60af16178c813 100644 --- a/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go +++ b/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go @@ -18,74 +18,127 @@ package monitoring import ( "fmt" - "k8s.io/api/core/v1" + gcm "google.golang.org/api/monitoring/v3" + corev1 "k8s.io/api/core/v1" + extensions "k8s.io/api/extensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/test/e2e/framework" ) var ( - CustomMetricName = "foo-metric" - UnusedMetricName = "unused-metric" - MetricValue1 = int64(448) - MetricValue2 = int64(446) + CustomMetricName = "foo-metric" + UnusedMetricName = "unused-metric" + CustomMetricValue = int64(448) + UnusedMetricValue = int64(446) +) - SDExporterPod1 = &v1.Pod{ +// StackdriverExporterDeployment is a Deployment of simple application that exports a metric of +// fixed value to Stackdriver in a loop. +func StackdriverExporterDeployment(name, namespace string, replicas int32, metricValue int64) *extensions.Deployment { + return &extensions.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: "sd-exporter-1", - Namespace: "default", - Labels: map[string]string{ - "name": "sd-exporter", - }, + Name: name, + Namespace: namespace, }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "sd-exporter", - Image: "gcr.io/google-containers/sd-dummy-exporter:v0.1.0", - ImagePullPolicy: v1.PullPolicy("Always"), - Command: []string{"/sd_dummy_exporter", "--pod-id=$(POD_ID)", "--metric-name=" + CustomMetricName, fmt.Sprintf("--metric-value=%v", MetricValue1)}, - Env: []v1.EnvVar{ - { - Name: "POD_ID", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "metadata.uid", - }, - }, - }, + Spec: extensions.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"name": name}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "name": name, }, - Ports: []v1.ContainerPort{{ContainerPort: 80}}, }, + Spec: stackdriverExporterPodSpec(CustomMetricName, metricValue), }, + Replicas: &replicas, }, } - SDExporterPod2 = &v1.Pod{ +} + +// StackdriverExporterPod is a Pod of simple application that exports a metric of fixed value to +// Stackdriver in a loop. +func StackdriverExporterPod(podName, namespace, podLabel, metricName string, metricValue int64) *corev1.Pod { + return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: "sd-exporter-2", - Namespace: "default", + Name: podName, + Namespace: namespace, Labels: map[string]string{ - "name": "sd-exporter", + "name": podLabel, }, }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "sd-exporter", - Image: "gcr.io/google-containers/sd-dummy-exporter:v0.1.0", - ImagePullPolicy: v1.PullPolicy("Always"), - Command: []string{"/sd_dummy_exporter", "--pod-id=$(POD_ID)", "--metric-name=" + CustomMetricName, fmt.Sprintf("--metric-value=%v", MetricValue2)}, - Env: []v1.EnvVar{ - { - Name: "POD_ID", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "metadata.uid", - }, + Spec: stackdriverExporterPodSpec(metricName, metricValue), + } +} + +func stackdriverExporterPodSpec(metricName string, metricValue int64) corev1.PodSpec { + return corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "stackdriver-exporter", + Image: "gcr.io/google-containers/sd-dummy-exporter:v0.1.0", + ImagePullPolicy: corev1.PullPolicy("Always"), + Command: []string{"/sd_dummy_exporter", "--pod-id=$(POD_ID)", "--metric-name=" + metricName, fmt.Sprintf("--metric-value=%v", metricValue)}, + Env: []corev1.EnvVar{ + { + Name: "POD_ID", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.uid", }, }, }, - Ports: []v1.ContainerPort{{ContainerPort: 80}}, }, + Ports: []corev1.ContainerPort{{ContainerPort: 80}}, }, }, } -) +} + +// CreateAdapter creates Custom Metrics - Stackdriver adapter. +func CreateAdapter() error { + stat, err := framework.RunKubectl("create", "-f", "https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/adapter-beta.yaml") + framework.Logf(stat) + return err +} + +// CreateDescriptors creates descriptors for metrics: CustomMetricName and UnusedMetricName. +func CreateDescriptors(service *gcm.Service, projectId string) error { + _, err := service.Projects.MetricDescriptors.Create(fmt.Sprintf("projects/%s", projectId), &gcm.MetricDescriptor{ + Name: CustomMetricName, + ValueType: "INT64", + Type: "custom.googleapis.com/" + CustomMetricName, + MetricKind: "GAUGE", + }).Do() + if err != nil { + return err + } + _, err = service.Projects.MetricDescriptors.Create(fmt.Sprintf("projects/%s", projectId), &gcm.MetricDescriptor{ + Name: UnusedMetricName, + ValueType: "INT64", + Type: "custom.googleapis.com/" + UnusedMetricName, + MetricKind: "GAUGE", + }).Do() + return err +} + +// CleanupDescriptors deletes descriptors for metrics: CustomMetricName and UnusedMetricName. +// TODO: Cleanup time series as well +func CleanupDescriptors(service *gcm.Service, projectId string) { + _, err := service.Projects.MetricDescriptors.Delete(fmt.Sprintf("projects/%s/metricDescriptors/custom.googleapis.com/%s", projectId, CustomMetricName)).Do() + if err != nil { + framework.Logf("Failed to delete descriptor for metric '%s': %v", CustomMetricName, err) + } + _, err = service.Projects.MetricDescriptors.Delete(fmt.Sprintf("projects/%s/metricDescriptors/custom.googleapis.com/%s", projectId, UnusedMetricName)).Do() + if err != nil { + framework.Logf("Failed to delete descriptor for metric '%s': %v", CustomMetricName, err) + } +} + +// CleanupAdapter deletes Custom Metrics - Stackdriver adapter deployments. +func CleanupAdapter() error { + stat, err := framework.RunKubectl("delete", "-f", "https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/adapter-beta.yaml") + framework.Logf(stat) + return err +} diff --git a/test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go b/test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go index 3aaa0ab27e623..f21e613656022 100644 --- a/test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go +++ b/test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go @@ -18,7 +18,6 @@ package monitoring import ( "context" - "fmt" "time" "golang.org/x/oauth2/google" @@ -26,7 +25,6 @@ import ( . "github.com/onsi/ginkgo" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kubernetes/test/e2e/framework" instrumentation "k8s.io/kubernetes/test/e2e/instrumentation/common" gcm "google.golang.org/api/monitoring/v3" @@ -35,9 +33,16 @@ import ( "k8s.io/apimachinery/pkg/selection" "k8s.io/client-go/discovery" kubeaggrcs "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" + "k8s.io/kubernetes/test/e2e/framework" customclient "k8s.io/metrics/pkg/client/custom_metrics" ) +const ( + stackdriverExporterPod1 = "stackdriver-exporter-1" + stackdriverExporterPod2 = "stackdriver-exporter-2" + stackdriverExporterLabel = "stackdriver-exporter" +) + var _ = instrumentation.SIGDescribe("Stackdriver Monitoring", func() { BeforeEach(func() { framework.SkipUnlessProviderIs("gce", "gke") @@ -49,7 +54,7 @@ var _ = instrumentation.SIGDescribe("Stackdriver Monitoring", func() { var customMetricsClient customclient.CustomMetricsClient var discoveryClient *discovery.DiscoveryClient - It("should run Custom Metrics - Stackdriver Adapter [Feature:StackdriverMonitoring]", func() { + It("should run Custom Metrics - Stackdriver Adapter [Feature:StackdriverCustomMetrics]", func() { kubeClient = f.ClientSet kubeAggrClient = f.AggregatorClient config, err := framework.LoadConfig() @@ -58,27 +63,27 @@ var _ = instrumentation.SIGDescribe("Stackdriver Monitoring", func() { } customMetricsClient = customclient.NewForConfigOrDie(config) discoveryClient = discovery.NewDiscoveryClientForConfigOrDie(config) - testAdapter(f, kubeClient, kubeAggrClient, customMetricsClient, discoveryClient) + testAdapter(f, kubeClient, customMetricsClient, discoveryClient) }) }) -func testAdapter(f *framework.Framework, kubeClient clientset.Interface, kubeAggrClient kubeaggrcs.Interface, customMetricsClient customclient.CustomMetricsClient, discoveryClient *discovery.DiscoveryClient) { +func testAdapter(f *framework.Framework, kubeClient clientset.Interface, customMetricsClient customclient.CustomMetricsClient, discoveryClient *discovery.DiscoveryClient) { projectId := framework.TestContext.CloudConfig.ProjectID ctx := context.Background() client, err := google.DefaultClient(ctx, gcm.CloudPlatformScope) - // Hack for running tests locally + // Hack for running tests locally, needed to authenticate in Stackdriver // If this is your use case, create application default credentials: // $ gcloud auth application-default login // and uncomment following lines (comment out the two lines above): /* - ts, err := google.DefaultTokenSource(oauth2.NoContext) - framework.Logf("Couldn't get application default credentials, %v", err) - if err != nil { - framework.Failf("Error accessing application default credentials, %v", err) - } - client := oauth2.NewClient(oauth2.NoContext, ts) + ts, err := google.DefaultTokenSource(oauth2.NoContext) + framework.Logf("Couldn't get application default credentials, %v", err) + if err != nil { + framework.Failf("Error accessing application default credentials, %v", err) + } + client := oauth2.NewClient(oauth2.NoContext, ts) */ gcmService, err := gcm.New(client) @@ -86,29 +91,29 @@ func testAdapter(f *framework.Framework, kubeClient clientset.Interface, kubeAgg framework.Failf("Failed to create gcm service, %v", err) } - framework.ExpectNoError(err) - // Set up a cluster: create a custom metric and set up k8s-sd adapter - err = createDescriptors(gcmService, projectId) + err = CreateDescriptors(gcmService, projectId) if err != nil { framework.Failf("Failed to create metric descriptor: %s", err) } - defer cleanupDescriptors(gcmService, projectId) + defer CleanupDescriptors(gcmService, projectId) - err = createAdapter() + err = CreateAdapter() if err != nil { framework.Failf("Failed to set up: %s", err) } - defer cleanupAdapter() + defer CleanupAdapter() // Run application that exports the metric - err = createSDExporterPod(kubeClient) + err = createSDExporterPods(f, kubeClient) if err != nil { - framework.Failf("Failed to create sd-exporter pod: %s", err) + framework.Failf("Failed to create stackdriver-exporter pod: %s", err) } - defer cleanupSDExporterPod(kubeClient) + defer cleanupSDExporterPod(f, kubeClient) // Wait a short amount of time to create a pod and export some metrics + // TODO: add some events to wait for instead of fixed amount of time + // i.e. pod creation, first time series exported time.Sleep(60 * time.Second) // Verify responses from Custom Metrics API @@ -121,18 +126,18 @@ func testAdapter(f *framework.Framework, kubeClient clientset.Interface, kubeAgg framework.Failf("Unexpected metric %s. Only metric %s should be supported", resource.Name, CustomMetricName) } } - value, err := customMetricsClient.NamespacedMetrics("default").GetForObject(schema.GroupKind{Group: "", Kind: "Pod"}, "sd-exporter-1", CustomMetricName) + value, err := customMetricsClient.NamespacedMetrics(f.Namespace.Name).GetForObject(schema.GroupKind{Group: "", Kind: "Pod"}, stackdriverExporterPod1, CustomMetricName) if err != nil { framework.Failf("Failed query: %s", err) } - if value.Value.Value() != MetricValue1 { - framework.Failf("Unexpected metric value for metric %s: expected %v but received %v", CustomMetricName, MetricValue1, value.Value) + if value.Value.Value() != CustomMetricValue { + framework.Failf("Unexpected metric value for metric %s: expected %v but received %v", CustomMetricName, CustomMetricValue, value.Value) } - filter, err := labels.NewRequirement("name", selection.Equals, []string{"sd-exporter"}) + filter, err := labels.NewRequirement("name", selection.Equals, []string{stackdriverExporterLabel}) if err != nil { framework.Failf("Couldn't create a label filter") } - values, err := customMetricsClient.NamespacedMetrics("default").GetForObjects(schema.GroupKind{Group: "", Kind: "Pod"}, labels.NewSelector().Add(*filter), CustomMetricName) + values, err := customMetricsClient.NamespacedMetrics(f.Namespace.Name).GetForObjects(schema.GroupKind{Group: "", Kind: "Pod"}, labels.NewSelector().Add(*filter), CustomMetricName) if err != nil { framework.Failf("Failed query: %s", err) } @@ -140,61 +145,29 @@ func testAdapter(f *framework.Framework, kubeClient clientset.Interface, kubeAgg framework.Failf("Expected results for exactly 2 pods, but %v results received", len(values.Items)) } for _, value := range values.Items { - if (value.DescribedObject.Name == SDExporterPod1.Name && value.Value.Value() != MetricValue1) || - (value.DescribedObject.Name == SDExporterPod2.Name && value.Value.Value() != MetricValue2) { + if (value.DescribedObject.Name == stackdriverExporterPod1 && value.Value.Value() != CustomMetricValue) || + (value.DescribedObject.Name == stackdriverExporterPod2 && value.Value.Value() != UnusedMetricValue) { framework.Failf("Unexpected metric value for metric %s and pod %s: %v", CustomMetricName, value.DescribedObject.Name, value.Value.Value()) } } - - framework.ExpectNoError(err) } -func createDescriptors(service *gcm.Service, projectId string) error { - _, err := service.Projects.MetricDescriptors.Create(fmt.Sprintf("projects/%s", projectId), &gcm.MetricDescriptor{ - Name: CustomMetricName, - ValueType: "INT64", - Type: "custom.googleapis.com/" + CustomMetricName, - MetricKind: "GAUGE", - }).Do() +func cleanupSDExporterPod(f *framework.Framework, cs clientset.Interface) { + err := cs.Core().Pods(f.Namespace.Name).Delete(stackdriverExporterPod1, &metav1.DeleteOptions{}) if err != nil { - return err + framework.Logf("Failed to delete %s pod: %v", stackdriverExporterPod1, err) + } + err = cs.Core().Pods(f.Namespace.Name).Delete(stackdriverExporterPod2, &metav1.DeleteOptions{}) + if err != nil { + framework.Logf("Failed to delete %s pod: %v", stackdriverExporterPod2, err) } - _, err = service.Projects.MetricDescriptors.Create(fmt.Sprintf("projects/%s", projectId), &gcm.MetricDescriptor{ - Name: UnusedMetricName, - ValueType: "INT64", - Type: "custom.googleapis.com/" + UnusedMetricName, - MetricKind: "GAUGE", - }).Do() - return err } -func createSDExporterPod(cs clientset.Interface) error { - _, err := cs.Core().Pods("default").Create(SDExporterPod1) +func createSDExporterPods(f *framework.Framework, cs clientset.Interface) error { + _, err := cs.Core().Pods(f.Namespace.Name).Create(StackdriverExporterPod(stackdriverExporterPod1, f.Namespace.Name, stackdriverExporterLabel, CustomMetricName, CustomMetricValue)) if err != nil { return err } - _, err = cs.Core().Pods("default").Create(SDExporterPod2) - return err -} - -func createAdapter() error { - stat, err := framework.RunKubectl("create", "-f", "https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/adapter-beta.yaml") - framework.Logf(stat) - return err -} - -func cleanupDescriptors(service *gcm.Service, projectId string) { - _, _ = service.Projects.MetricDescriptors.Delete(fmt.Sprintf("projects/%s/metricDescriptors/custom.googleapis.com/%s", projectId, CustomMetricName)).Do() - _, _ = service.Projects.MetricDescriptors.Delete(fmt.Sprintf("projects/%s/metricDescriptors/custom.googleapis.com/%s", projectId, UnusedMetricName)).Do() -} - -func cleanupSDExporterPod(cs clientset.Interface) { - _ = cs.Core().Pods("default").Delete("sd-exporter-1", &metav1.DeleteOptions{}) - _ = cs.Core().Pods("default").Delete("sd-exporter-2", &metav1.DeleteOptions{}) -} - -func cleanupAdapter() error { - stat, err := framework.RunKubectl("delete", "-f", "https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/adapter-beta.yaml") - framework.Logf(stat) + _, err = cs.Core().Pods(f.Namespace.Name).Create(StackdriverExporterPod(stackdriverExporterPod2, f.Namespace.Name, stackdriverExporterLabel, UnusedMetricName, UnusedMetricValue)) return err } diff --git a/test/e2e/instrumentation/monitoring/stackdriver.go b/test/e2e/instrumentation/monitoring/stackdriver.go index 6923ce2e1fcd9..c11a92e6522e7 100644 --- a/test/e2e/instrumentation/monitoring/stackdriver.go +++ b/test/e2e/instrumentation/monitoring/stackdriver.go @@ -82,12 +82,12 @@ func testStackdriverMonitoring(f *framework.Framework, pods, allPodsCPU int, per // $ gcloud auth application-default login // and uncomment following lines (comment out the two lines above): (DON'T set the env var below) /* - ts, err := google.DefaultTokenSource(oauth2.NoContext) - framework.Logf("Couldn't get application default credentials, %v", err) - if err != nil { - framework.Failf("Error accessing application default credentials, %v", err) - } - client := oauth2.NewClient(oauth2.NoContext, ts) + ts, err := google.DefaultTokenSource(oauth2.NoContext) + framework.Logf("Couldn't get application default credentials, %v", err) + if err != nil { + framework.Failf("Error accessing application default credentials, %v", err) + } + client := oauth2.NewClient(oauth2.NoContext, ts) */ gcmService, err := gcm.New(client) From f3fddaedd05931ac234565b1da22f3167e1253bb Mon Sep 17 00:00:00 2001 From: Karol Wychowaniec Date: Thu, 28 Sep 2017 16:01:27 +0200 Subject: [PATCH 3/3] Enable HorizontalPodAutoscalerUseRESTClients option --- cluster/gce/gci/configure-helper.sh | 4 ++++ .../app/options/options.go | 2 +- .../app/options/options_test.go | 19 ++++++++++--------- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 2011dd6fee308..0cc04fa47393c 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -1602,6 +1602,10 @@ function start-kube-controller-manager { if [[ -n "${CLUSTER_SIGNING_DURATION:-}" ]]; then params+=" --experimental-cluster-signing-duration=$CLUSTER_SIGNING_DURATION" fi + # disable using HPA metrics REST clients if metrics-server isn't enabled + if [[ "${ENABLE_METRICS_SERVER:-}" != "true" ]]; then + params+=" --horizontal-pod-autoscaler-use-rest-clients=false" + fi local -r kube_rc_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-controller-manager.docker_tag) local container_env="" diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index 12f6264dfb0a6..dddfce2d9707f 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -115,7 +115,7 @@ func NewCMServer() *CMServer { ClusterSigningDuration: metav1.Duration{Duration: helpers.OneYear}, ReconcilerSyncLoopPeriod: metav1.Duration{Duration: 60 * time.Second}, EnableTaintManager: true, - HorizontalPodAutoscalerUseRESTClients: false, + HorizontalPodAutoscalerUseRESTClients: true, }, } s.LeaderElection.LeaderElect = true diff --git a/cmd/kube-controller-manager/app/options/options_test.go b/cmd/kube-controller-manager/app/options/options_test.go index ce5beff0ef814..78addd5c86801 100644 --- a/cmd/kube-controller-manager/app/options/options_test.go +++ b/cmd/kube-controller-manager/app/options/options_test.go @@ -192,15 +192,16 @@ func TestAddFlags(t *testing.T) { {Group: "apiregistration.k8s.io", Resource: "apiservices"}, {Group: "apiextensions.k8s.io", Resource: "customresourcedefinitions"}, }, - NodeEvictionRate: 0.2, - SecondaryNodeEvictionRate: 0.05, - LargeClusterSizeThreshold: 100, - UnhealthyZoneThreshold: 0.6, - DisableAttachDetachReconcilerSync: true, - ReconcilerSyncLoopPeriod: metav1.Duration{Duration: 30 * time.Second}, - Controllers: []string{"foo", "bar"}, - EnableTaintManager: false, - UseServiceAccountCredentials: true, + NodeEvictionRate: 0.2, + SecondaryNodeEvictionRate: 0.05, + LargeClusterSizeThreshold: 100, + UnhealthyZoneThreshold: 0.6, + DisableAttachDetachReconcilerSync: true, + ReconcilerSyncLoopPeriod: metav1.Duration{Duration: 30 * time.Second}, + Controllers: []string{"foo", "bar"}, + EnableTaintManager: false, + HorizontalPodAutoscalerUseRESTClients: true, + UseServiceAccountCredentials: true, }, Kubeconfig: "/kubeconfig", Master: "192.168.4.20",