feat: expose provider metrics in the Helm chart

cmd/ratify-gatekeeper-provider/main.go

@@ -28,11 +28,14 @@ import (
 	"github.com/notaryproject/ratify/v2/internal/httpserver"
 	"github.com/notaryproject/ratify/v2/internal/manager"
 	"github.com/notaryproject/ratify/v2/pkg/common"
+	"github.com/notaryproject/ratify/v2/pkg/metrics"
 	"github.com/sirupsen/logrus"
 )
 
 var startManagerFunc = manager.StartManager
 
+var initMetricsFunc = metrics.InitMetricsExporter
+
 // main is the entry point for the Ratify server.
 func main() {
 	common.SetLoggingLevelFromEnv(logrus.StandardLogger())
@@ -54,6 +57,8 @@ type options struct {
 	disableCRDManager    bool
 	verifyTimeout        time.Duration
 	mutateTimeout        time.Duration
+	enableMetrics        bool
+	metricsPort          int
 }
 
 func parse() *options {
@@ -69,6 +74,8 @@ func parse() *options {
 	flag.BoolVar(&opts.disableCertRotation, "disable-cert-rotation", false, "Disable certificate rotation")
 	flag.BoolVar(&opts.disableMutation, "disable-mutation", false, "Disable mutation wehbook")
 	flag.BoolVar(&opts.disableCRDManager, "disable-crd-manager", false, "Disable CRD manager for Gatekeeper provider")
+	flag.BoolVar(&opts.enableMetrics, "enable-metrics", false, "Enable the Prometheus metrics exporter")
+	flag.IntVar(&opts.metricsPort, "metrics-port", 8888, "Port for the Prometheus /metrics endpoint")
 
 	flag.Parse()
 	logrus.Infof("Starting Ratify with options: %+v", opts)
@@ -79,6 +86,11 @@ func startRatify(opts *options) error {
 	if len(opts.httpServerAddress) == 0 {
 		return errors.New("HTTP server address is required")
 	}
+	if opts.enableMetrics {
+		if err := initMetricsFunc("prometheus", opts.metricsPort); err != nil {
+			logrus.Errorf("failed to initialize metrics exporter: %v", err)
+		}
+	}
 	var certRotatorReady chan struct{}
 	if !opts.disableCertRotation {
 		certRotatorReady = make(chan struct{})

cmd/ratify-gatekeeper-provider/main_test.go

@@ -17,6 +17,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"net"
 	"os"
@@ -26,6 +27,9 @@ import (
 )
 
 func TestMain_FailedStartingRatify(t *testing.T) {
+	origInitMetrics := initMetricsFunc
+	defer func() { initMetricsFunc = origInitMetrics }()
+	initMetricsFunc = func(string, int) error { return nil }
 	args := []string{
 		"-config=config.json",
 		"-cert-file=cert.pem",
@@ -61,6 +65,8 @@ func TestParse(t *testing.T) {
 				"-cert-file=cert.pem",
 				"-key-file=key.pem",
 				"-verify-timeout=10s",
+				"-enable-metrics",
+				"-metrics-port=9999",
 			},
 			expected: &options{
 				configFilePath:      "config.json",
@@ -70,6 +76,8 @@ func TestParse(t *testing.T) {
 				keyFile:             "key.pem",
 				verifyTimeout:       10 * time.Second,
 				mutateTimeout:       2 * time.Second,
+				enableMetrics:       true,
+				metricsPort:         9999,
 			},
 		},
 		{
@@ -82,6 +90,7 @@ func TestParse(t *testing.T) {
 				healthServerAddress: ":9099",
 				verifyTimeout:       30 * time.Second,
 				mutateTimeout:       10 * time.Second,
+				metricsPort:         8888,
 			},
 		},
 		{
@@ -91,6 +100,7 @@ func TestParse(t *testing.T) {
 				healthServerAddress: ":9099",
 				verifyTimeout:       5 * time.Second,
 				mutateTimeout:       2 * time.Second,
+				metricsPort:         8888,
 			},
 		},
 	}
@@ -114,11 +124,20 @@ func TestParse(t *testing.T) {
 }
 
 func TestStartRatify(t *testing.T) {
+	origStartManager := startManagerFunc
+	origInitMetrics := initMetricsFunc
+	defer func() {
+		startManagerFunc = origStartManager
+		initMetricsFunc = origInitMetrics
+	}()
 	startManagerFunc = func(_, _ chan struct{}, _, _ bool) {}
+	errMetricsInit := errors.New("metrics boom")
+	initMetricsFunc = func(string, int) error { return errMetricsInit }
 	tests := []struct {
 		name        string
 		opts        *options
 		expectError bool
+		notError    error
 	}{
 		{
 			name: "missing http server address",
@@ -141,6 +160,20 @@ func TestStartRatify(t *testing.T) {
 			},
 			expectError: true,
 		},
+		{
+			name: "metrics init failure is non-fatal",
+			opts: &options{
+				httpServerAddress:   ":8080",
+				configFilePath:      "config.yaml",
+				certFile:            "cert.pem",
+				disableCertRotation: true,
+				disableCRDManager:   true,
+				enableMetrics:       true,
+				metricsPort:         8888,
+			},
+			expectError: true,
+			notError:    errMetricsInit,
+		},
 	}
 
 	for _, tt := range tests {
@@ -149,6 +182,9 @@ func TestStartRatify(t *testing.T) {
 			if (err != nil) != tt.expectError {
 				t.Errorf("startRatify() error = %v, expectError %v", err, tt.expectError)
 			}
+			if tt.notError != nil && errors.Is(err, tt.notError) {
+				t.Errorf("startRatify() returned the metrics init error %v; it should be non-fatal", err)
+			}
 		})
 	}
 }

deployments/ratify-gatekeeper-provider/README.md

@@ -45,6 +45,8 @@ Values marked `# DEPRECATED` in the `values.yaml` as well as **DEPRECATED** in t
 | `resources.requests.memory`               | Memory request for the provider container.                                                                                                                                                           | `512Mi`                                         |
 | `resources.limits.cpu`                    | CPU limit for the provider container.                                                                                                                                                                | `1000m`                                         |
 | `resources.limits.memory`                 | Memory limit for the provider container.                                                                                                                                                              | `512Mi`                                         |
+| `metrics.enabled`                         | Expose the Prometheus `/metrics` endpoint (adds the container port and pod scrape annotations).                                                                                                      | `true`                                          |
+| `metrics.port`                            | Port for the `/metrics` endpoint.                                                                                                                                                                    | `8888`                                          |
 | `notation.scopes`                         | Scopes that Notation verifier is applicable for. See [Notation trust policy](https://github.com/notaryproject/specifications/blob/main/specs/trust-store-trust-policy.md#trust-policy).                 | `[]`                                            |
 | `notation.trustedIdentities`              | List of trusted identities for Notation verifier. See [Notation trust policy](https://github.com/notaryproject/specifications/blob/main/specs/trust-store-trust-policy.md#trust-policy).                | `[]`                                            |
 | `notation.certs`                          | List of trusted root certificates for Notation verifier.                                                                                                                                              | `[]`                                            |

deployments/ratify-gatekeeper-provider/templates/deployment.yaml

@@ -11,6 +11,12 @@ spec:
       {{- include "ratify.selectorLabels" . | nindent 6 }}
   template:
     metadata:
+      {{- if .Values.metrics.enabled }}
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: {{ .Values.metrics.port | quote }}
+        prometheus.io/path: "/metrics"
+      {{- end }}
       labels:
         {{- include "ratify.selectorLabels" . | nindent 8 }}
         {{- if or (eq (index .Values.stores 0).credential.provider "azure") (eq (include "ratify.akvCertsProvided" .) "true") }}
@@ -66,11 +72,20 @@ spec:
             {{- if (lookup "v1" "Secret" .Release.Namespace "gatekeeper-webhook-server-cert") }}
             - "--gatekeeper-ca-cert-file=/usr/local/tls/client-ca/ca.crt"
             {{- end }}
+            {{- if .Values.metrics.enabled }}
+            - "--enable-metrics"
+            - "--metrics-port={{ .Values.metrics.port | int }}"
+            {{- end }}
           ports:
             - containerPort: 6001
             - containerPort: 9099
               name: healthz
               protocol: TCP
+            {{- if .Values.metrics.enabled }}
+            - containerPort: {{ .Values.metrics.port | int }}
+              name: metrics
+              protocol: TCP
+            {{- end }}
           livenessProbe:
             httpGet:
               path: /healthz

deployments/ratify-gatekeeper-provider/values.yaml

@@ -18,6 +18,10 @@ resources:
     cpu: 600m
     memory: 512Mi
 
+metrics:
+  enabled: true
+  port: 8888
+
 executor:
   scopes: []
   concurrency: 3

internal/httpserver/handlers.go

@@ -23,14 +23,18 @@ import (
 	"io"
 	"net/http"
 	"strings"
+	"time"
 
+	"github.com/notaryproject/ratify/v2/pkg/metrics"
 	"github.com/open-policy-agent/frameworks/constraint/pkg/externaldata"
 	"github.com/sirupsen/logrus"
 	"oras.land/oras-go/v2/registry"
 )
 
 // verify handles the verification request from Gatekeeper.
 func (s *server) verify(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
+	start := time.Now()
+	defer func() { metrics.ReportVerificationRequest(ctx, time.Since(start).Milliseconds()) }()
 	defer r.Body.Close()
 	body, err := io.ReadAll(r.Body)
 	if err != nil {
@@ -84,6 +88,8 @@ func (s *server) verify(ctx context.Context, w http.ResponseWriter, r *http.Requ
 
 // mutate handles the mutation request from Gatekeeper.
 func (s *server) mutate(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
+	start := time.Now()
+	defer func() { metrics.ReportMutationRequest(ctx, time.Since(start).Milliseconds()) }()
 	defer r.Body.Close()
 	body, err := io.ReadAll(r.Body)
 	if err != nil {