Metrics (#70)

* enabling metrics Signed-off-by: raffaelespazzoli <[email protected]> * fixed selector Signed-off-by: raffaelespazzoli <[email protected]> * fixed manifest generation Signed-off-by: raffaelespazzoli <[email protected]> * fixed helm template label Signed-off-by: raffaelespazzoli <[email protected]>
redhat-cop · Jun 2, 2021 · d8e44ac · d8e44ac
1 parent c7aab79
commit d8e44ac
Show file tree

Hide file tree

Showing 13 changed files with 114 additions and 20 deletions.
diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml
@@ -1,5 +1,5 @@
 # Adds namespace to all resources.
-namespace: egressip-ipam-operator-system
+namespace: egressip-ipam-operator
 
 # Value of this field is prepended to the
 # names of all resources, e.g. a deployment named
@@ -22,13 +22,13 @@ bases:
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
 #- ../certmanager
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
-#- ../prometheus
+- ../prometheus
 
 patchesStrategicMerge:
 # Protect the /metrics endpoint by putting it behind auth.
 # If you want your controller-manager to expose the /metrics
 # endpoint w/o any authn/z, please comment the following line.
-#- manager_auth_proxy_patch.yaml
+- manager_auth_proxy_patch.yaml
 
 # Mount the controller config file for loading manager configurations
 # through a ComponentConfig type
@@ -45,6 +45,19 @@ patchesStrategicMerge:
 
 # the following config is for teaching kustomize how to do var substitution
 vars:
+- name: METRICS_SERVICE_NAME
+  objref:
+    kind: Service
+    version: v1
+    name: controller-manager-metrics
+- name: METRICS_SERVICE_NAMESPACE
+  objref:
+    kind: Service
+    version: v1
+    name: controller-manager-metrics
+  fieldref:
+    fieldpath: metadata.namespace
+
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
 #- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
 #  objref:

diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml
@@ -16,6 +16,11 @@ spec:
         - "--upstream=http://127.0.0.1:8080/"
         - "--logtostderr=true"
         - "--v=10"
+        - "--tls-cert-file=/etc/certs/tls/tls.crt"
+        - "--tls-private-key-file=/etc/certs/tls/tls.key"
+        volumeMounts:
+          - mountPath: /etc/certs/tls
+            name: tls-cert         
         ports:
         - containerPort: 8443
           name: https
@@ -24,3 +29,8 @@ spec:
         - "--health-probe-bind-address=:8081"
         - "--metrics-bind-address=127.0.0.1:8080"
         - "--leader-elect"
+      volumes:
+      - name: tls-cert
+        secret:
+          defaultMode: 420
+          secretName: egressip-ipam-operator-certs
diff --git a/config/helmchart/kustomization.yaml b/config/helmchart/kustomization.yaml
@@ -14,3 +14,18 @@ namePrefix: egressip-ipam-operator-
 
 bases:
 - ../rbac
+- ../prometheus
+
+vars:
+- name: METRICS_SERVICE_NAME
+  objref:
+    kind: Service
+    version: v1
+    name: controller-manager-metrics
+- name: METRICS_SERVICE_NAMESPACE
+  objref:
+    kind: Service
+    version: v1
+    name: controller-manager-metrics
+  fieldref:
+    fieldpath: metadata.namespace
diff --git a/config/helmchart/templates/manager.yaml b/config/helmchart/templates/manager.yaml
@@ -17,13 +17,29 @@ spec:
     {{- end }}
       labels:
         {{- include "egressip-ipam-operator.selectorLabels" . | nindent 8 }}
+        operator: egressip-ipam-operator
     spec:
       serviceAccountName: egressip-ipam-operator-controller-manager
       {{- with .Values.imagePullSecrets }}
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
       {{- end }} 
       containers:
+      - args:
+        - --secure-listen-address=0.0.0.0:8443
+        - --upstream=http://127.0.0.1:8080/
+        - --logtostderr=true
+        - --tls-cert-file=/etc/certs/tls/tls.crt
+        - --tls-private-key-file=/etc/certs/tls/tls.key
+        - --v=10
+        image: quay.io/coreos/kube-rbac-proxy:v0.5.0
+        name: kube-rbac-proxy
+        ports:
+        - containerPort: 8443
+          name: https
+        volumeMounts:
+        - mountPath: /etc/certs/tls
+          name: tls-cert        
       - command:
         - /manager
         args:
@@ -56,4 +72,9 @@ spec:
       {{- with .Values.tolerations }}
       tolerations:
         {{- toYaml . | nindent 8 }}
-      {{- end }}
+      {{- end }}
+      volumes:
+      - name: tls-cert
+        secret:
+          defaultMode: 420
+          secretName: egressip-ipam-operator-certs      
diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml
@@ -3,6 +3,7 @@ kind: Namespace
 metadata:
   labels:
     control-plane: controller-manager
+    openshift.io/cluster-monitoring: "true"    
   name: system
 ---
 apiVersion: apps/v1
@@ -11,16 +12,16 @@ metadata:
   name: controller-manager
   namespace: system
   labels:
-    control-plane: controller-manager
+    operator: egressip-ipam-operator
 spec:
   selector:
     matchLabels:
-      control-plane: controller-manager
+      operator: egressip-ipam-operator
   replicas: 1
   template:
     metadata:
       labels:
-        control-plane: controller-manager
+        operator: egressip-ipam-operator
     spec:
       serviceAccountName: controller-manager
       containers:

diff --git a/config/manifests/bases/egressip-ipam-operator.clusterserviceversion.yaml b/config/manifests/bases/egressip-ipam-operator.clusterserviceversion.yaml
@@ -8,7 +8,7 @@ metadata:
     certified: "false"
     containerImage: quay.io/redhat-cop/egressip-ipam-operator
     createdAt: 03/13/2020
-    description: This operator provides automation around manageing egressIPs.
+    description: This operator provides automation around managing egressIPs.
     operatorframework.io/suggested-namespace: egressip-ipam-operator
     repository: https://github.com/redhat-cop/egressip-ipam-operator
     support: Best Effort

diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml
@@ -1,2 +1,5 @@
 resources:
 - monitor.yaml
+
+configurations:
+- kustomizeconfig.yaml
diff --git a/config/prometheus/kustomizeconfig.yaml b/config/prometheus/kustomizeconfig.yaml
@@ -0,0 +1,4 @@
+---
+varReference:
+- path: spec/endpoints/tlsConfig/serverName
+  kind: ServiceMonitor
diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml
@@ -4,13 +4,18 @@ apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
   labels:
-    control-plane: controller-manager
+    operator: egressip-ipam-operator
   name: controller-manager-metrics-monitor
   namespace: system
 spec:
   endpoints:
-    - path: /metrics
+    - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+      interval: 30s
       port: https
+      scheme: https
+      tlsConfig:
+        caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+        serverName: $(METRICS_SERVICE_NAME).$(METRICS_SERVICE_NAMESPACE).svc
   selector:
     matchLabels:
-      control-plane: controller-manager
+      operator: egressip-ipam-operator
diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml
@@ -2,13 +2,15 @@ apiVersion: v1
 kind: Service
 metadata:
   labels:
-    control-plane: controller-manager
-  name: controller-manager-metrics-service
+    operator: egressip-ipam-operator
+  annotations:
+    service.alpha.openshift.io/serving-cert-secret-name: egressip-ipam-operator-certs
+  name: controller-manager-metrics
   namespace: system
 spec:
   ports:
   - name: https
     port: 8443
     targetPort: https
   selector:
-    control-plane: controller-manager
+    operator: egressip-ipam-operator
diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml
@@ -7,7 +7,7 @@ resources:
 # Comment the following 4 lines if you want to disable
 # the auth proxy (https://github.com/brancz/kube-rbac-proxy)
 # which protects your /metrics endpoint.
-# - auth_proxy_service.yaml
-# - auth_proxy_role.yaml
-# - auth_proxy_role_binding.yaml
-# - auth_proxy_client_clusterrole.yaml
+- auth_proxy_service.yaml
+- auth_proxy_role.yaml
+- auth_proxy_role_binding.yaml
+- auth_proxy_client_clusterrole.yaml
diff --git a/controllers/egressipam/azure/azure.go b/controllers/egressipam/azure/azure.go
@@ -1,3 +1,4 @@
+// +kubebuilder:skip
 package azure
 
 import (

diff --git a/readme.md b/readme.md
@@ -241,6 +241,14 @@ helm repo update
 helm upgrade egressip-ipam-operator egressip-ipam-operator/egressip-ipam-operator
 ```
 
+## Metrics
+
+Prometheus compatible metrics are exposed by the Operator and can be integrated into OpenShift's default cluster monitoring. To enable OpenShift cluster monitoring, label the namespace the operator is deployed in with the label `openshift.io/cluster-monitoring="true"`.
+
+```shell
+oc label namespace <namespace> openshift.io/cluster-monitoring="true"
+```
+
 ## Development
 
 ## Running the operator locally
@@ -249,7 +257,7 @@ helm upgrade egressip-ipam-operator egressip-ipam-operator/egressip-ipam-operato
 make install
 oc new-project egressip-ipam-operator-local
 kustomize build ./config/local-development | oc apply -f - -n egressip-ipam-operator-local
-export token=$(oc serviceaccounts get-token 'egressip-ipam-operator-controller-manager' -n egressip-ipam-operator-local)
+export token=$(oc serviceaccounts get-token 'egressip-ipam-controller-manager' -n egressip-ipam-operator-local)
 export NAMESPACE=egressip-ipam-operator-local
 oc login --token ${token}
 make run ENABLE_WEBHOOKS=false
@@ -282,6 +290,7 @@ kubectl delete -f charts/egressip-ipam-operator/crds/crds.yaml
 
 ```shell
 export repo=raffaelespazzoli #replace with yours
+docker login quay.io/$repo
 make docker-build IMG=quay.io/$repo/egressip-ipam-operator:latest
 make docker-push IMG=quay.io/$repo/egressip-ipam-operator:latest
 ```
@@ -293,9 +302,10 @@ make manifests
 make bundle IMG=quay.io/$repo/egressip-ipam-operator:latest
 operator-sdk bundle validate ./bundle --select-optional name=operatorhub
 make bundle-build BUNDLE_IMG=quay.io/$repo/egressip-ipam-operator-bundle:latest
-podman push quay.io/$repo/egressip-ipam-operator-bundle:latest
+docker push quay.io/$repo/egressip-ipam-operator-bundle:latest
 operator-sdk bundle validate quay.io/$repo/egressip-ipam-operator-bundle:latest --select-optional name=operatorhub
 oc new-project egressip-ipam-operator
+oc label namespace egressip-ipam-operator openshift.io/cluster-monitoring="true"
 operator-sdk cleanup egressip-ipam-operator -n egressip-ipam-operator
 operator-sdk run bundle --install-mode AllNamespaces -n egressip-ipam-operator quay.io/$repo/egressip-ipam-operator-bundle:latest
 ```
@@ -359,6 +369,15 @@ oc apply -f test/egressIPAM-Azure.yaml
 oc apply -f test/namespace-Azure.yaml
 ```
 
+#### Testing metrics
+
+```sh
+export operatorNamespace=egressip-ipam-operator-local # or egressip-ipam-operator
+oc label namespace ${operatorNamespace} openshift.io/cluster-monitoring="true"
+oc rsh -n openshift-monitoring -c prometheus prometheus-k8s-0 /bin/bash
+curl -v -s -k -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" https://resource-locker-operator-controller-manager-metrics.${operatorNamespace}.svc.cluster.local:8443/metrics
+```
+
 ## Releasing
 
 ```shell