From deaa3051600e6f3b35bc83a1668c7698c2fab8cf Mon Sep 17 00:00:00 2001 From: Charles Wu Date: Thu, 30 Jul 2026 10:03:22 +1000 Subject: [PATCH 1/2] feat: expose Prometheus metrics and instrument requests The metrics package (OTel + Prometheus, ported from v1) was never wired into the gatekeeper provider: the exporter was never initialized and no request duration was recorded, so the provider emitted no metrics and served no /metrics endpoint. Initialize the Prometheus exporter on startup behind a new --metrics-port flag (default 8888, 0 disables) and record verification/mutation request durations in the verify/mutate handlers. Exporter init failures are logged but non-fatal. Signed-off-by: Charles Wu --- cmd/ratify-gatekeeper-provider/main.go | 12 +++++++ cmd/ratify-gatekeeper-provider/main_test.go | 36 +++++++++++++++++++++ internal/httpserver/handlers.go | 6 ++++ 3 files changed, 54 insertions(+) diff --git a/cmd/ratify-gatekeeper-provider/main.go b/cmd/ratify-gatekeeper-provider/main.go index 03be81636..a6cc19e24 100644 --- a/cmd/ratify-gatekeeper-provider/main.go +++ b/cmd/ratify-gatekeeper-provider/main.go @@ -28,11 +28,14 @@ import ( "github.com/notaryproject/ratify/v2/internal/httpserver" "github.com/notaryproject/ratify/v2/internal/manager" "github.com/notaryproject/ratify/v2/pkg/common" + "github.com/notaryproject/ratify/v2/pkg/metrics" "github.com/sirupsen/logrus" ) var startManagerFunc = manager.StartManager +var initMetricsFunc = metrics.InitMetricsExporter + // main is the entry point for the Ratify server. func main() { common.SetLoggingLevelFromEnv(logrus.StandardLogger()) @@ -54,6 +57,8 @@ type options struct { disableCRDManager bool verifyTimeout time.Duration mutateTimeout time.Duration + enableMetrics bool + metricsPort int } func parse() *options { @@ -69,6 +74,8 @@ func parse() *options { flag.BoolVar(&opts.disableCertRotation, "disable-cert-rotation", false, "Disable certificate rotation") flag.BoolVar(&opts.disableMutation, "disable-mutation", false, "Disable mutation wehbook") flag.BoolVar(&opts.disableCRDManager, "disable-crd-manager", false, "Disable CRD manager for Gatekeeper provider") + flag.BoolVar(&opts.enableMetrics, "enable-metrics", false, "Enable the Prometheus metrics exporter") + flag.IntVar(&opts.metricsPort, "metrics-port", 8888, "Port for the Prometheus /metrics endpoint") flag.Parse() logrus.Infof("Starting Ratify with options: %+v", opts) @@ -79,6 +86,11 @@ func startRatify(opts *options) error { if len(opts.httpServerAddress) == 0 { return errors.New("HTTP server address is required") } + if opts.enableMetrics { + if err := initMetricsFunc("prometheus", opts.metricsPort); err != nil { + logrus.Errorf("failed to initialize metrics exporter: %v", err) + } + } var certRotatorReady chan struct{} if !opts.disableCertRotation { certRotatorReady = make(chan struct{}) diff --git a/cmd/ratify-gatekeeper-provider/main_test.go b/cmd/ratify-gatekeeper-provider/main_test.go index 712a1a6eb..c55df3ebd 100644 --- a/cmd/ratify-gatekeeper-provider/main_test.go +++ b/cmd/ratify-gatekeeper-provider/main_test.go @@ -17,6 +17,7 @@ package main import ( "context" + "errors" "flag" "net" "os" @@ -26,6 +27,9 @@ import ( ) func TestMain_FailedStartingRatify(t *testing.T) { + origInitMetrics := initMetricsFunc + defer func() { initMetricsFunc = origInitMetrics }() + initMetricsFunc = func(string, int) error { return nil } args := []string{ "-config=config.json", "-cert-file=cert.pem", @@ -61,6 +65,8 @@ func TestParse(t *testing.T) { "-cert-file=cert.pem", "-key-file=key.pem", "-verify-timeout=10s", + "-enable-metrics", + "-metrics-port=9999", }, expected: &options{ configFilePath: "config.json", @@ -70,6 +76,8 @@ func TestParse(t *testing.T) { keyFile: "key.pem", verifyTimeout: 10 * time.Second, mutateTimeout: 2 * time.Second, + enableMetrics: true, + metricsPort: 9999, }, }, { @@ -82,6 +90,7 @@ func TestParse(t *testing.T) { healthServerAddress: ":9099", verifyTimeout: 30 * time.Second, mutateTimeout: 10 * time.Second, + metricsPort: 8888, }, }, { @@ -91,6 +100,7 @@ func TestParse(t *testing.T) { healthServerAddress: ":9099", verifyTimeout: 5 * time.Second, mutateTimeout: 2 * time.Second, + metricsPort: 8888, }, }, } @@ -114,11 +124,20 @@ func TestParse(t *testing.T) { } func TestStartRatify(t *testing.T) { + origStartManager := startManagerFunc + origInitMetrics := initMetricsFunc + defer func() { + startManagerFunc = origStartManager + initMetricsFunc = origInitMetrics + }() startManagerFunc = func(_, _ chan struct{}, _, _ bool) {} + errMetricsInit := errors.New("metrics boom") + initMetricsFunc = func(string, int) error { return errMetricsInit } tests := []struct { name string opts *options expectError bool + notError error }{ { name: "missing http server address", @@ -141,6 +160,20 @@ func TestStartRatify(t *testing.T) { }, expectError: true, }, + { + name: "metrics init failure is non-fatal", + opts: &options{ + httpServerAddress: ":8080", + configFilePath: "config.yaml", + certFile: "cert.pem", + disableCertRotation: true, + disableCRDManager: true, + enableMetrics: true, + metricsPort: 8888, + }, + expectError: true, + notError: errMetricsInit, + }, } for _, tt := range tests { @@ -149,6 +182,9 @@ func TestStartRatify(t *testing.T) { if (err != nil) != tt.expectError { t.Errorf("startRatify() error = %v, expectError %v", err, tt.expectError) } + if tt.notError != nil && errors.Is(err, tt.notError) { + t.Errorf("startRatify() returned the metrics init error %v; it should be non-fatal", err) + } }) } } diff --git a/internal/httpserver/handlers.go b/internal/httpserver/handlers.go index f1836c5bd..9a9085eb5 100644 --- a/internal/httpserver/handlers.go +++ b/internal/httpserver/handlers.go @@ -23,7 +23,9 @@ import ( "io" "net/http" "strings" + "time" + "github.com/notaryproject/ratify/v2/pkg/metrics" "github.com/open-policy-agent/frameworks/constraint/pkg/externaldata" "github.com/sirupsen/logrus" "oras.land/oras-go/v2/registry" @@ -31,6 +33,8 @@ import ( // verify handles the verification request from Gatekeeper. func (s *server) verify(ctx context.Context, w http.ResponseWriter, r *http.Request) error { + start := time.Now() + defer func() { metrics.ReportVerificationRequest(ctx, time.Since(start).Milliseconds()) }() defer r.Body.Close() body, err := io.ReadAll(r.Body) if err != nil { @@ -84,6 +88,8 @@ func (s *server) verify(ctx context.Context, w http.ResponseWriter, r *http.Requ // mutate handles the mutation request from Gatekeeper. func (s *server) mutate(ctx context.Context, w http.ResponseWriter, r *http.Request) error { + start := time.Now() + defer func() { metrics.ReportMutationRequest(ctx, time.Since(start).Milliseconds()) }() defer r.Body.Close() body, err := io.ReadAll(r.Body) if err != nil { From 46cefd5820601220c045043b420e8165b61cd797 Mon Sep 17 00:00:00 2001 From: Charles Wu Date: Thu, 30 Jul 2026 13:23:41 +1000 Subject: [PATCH 2/2] feat: expose provider metrics in the Helm chart Wire the provider Prometheus /metrics endpoint into the chart: pass --metrics-enabled and --metrics-port when metrics.enabled is set, add the metrics container port, and add prometheus.io pod scrape annotations for annotation-based scraping. Signed-off-by: Charles Wu --- deployments/ratify-gatekeeper-provider/README.md | 2 ++ .../templates/deployment.yaml | 15 +++++++++++++++ .../ratify-gatekeeper-provider/values.yaml | 4 ++++ 3 files changed, 21 insertions(+) diff --git a/deployments/ratify-gatekeeper-provider/README.md b/deployments/ratify-gatekeeper-provider/README.md index 79eb47cf4..9c134eb62 100644 --- a/deployments/ratify-gatekeeper-provider/README.md +++ b/deployments/ratify-gatekeeper-provider/README.md @@ -45,6 +45,8 @@ Values marked `# DEPRECATED` in the `values.yaml` as well as **DEPRECATED** in t | `resources.requests.memory` | Memory request for the provider container. | `512Mi` | | `resources.limits.cpu` | CPU limit for the provider container. | `1000m` | | `resources.limits.memory` | Memory limit for the provider container. | `512Mi` | +| `metrics.enabled` | Expose the Prometheus `/metrics` endpoint (adds the container port and pod scrape annotations). | `true` | +| `metrics.port` | Port for the `/metrics` endpoint. | `8888` | | `notation.scopes` | Scopes that Notation verifier is applicable for. See [Notation trust policy](https://github.com/notaryproject/specifications/blob/main/specs/trust-store-trust-policy.md#trust-policy). | `[]` | | `notation.trustedIdentities` | List of trusted identities for Notation verifier. See [Notation trust policy](https://github.com/notaryproject/specifications/blob/main/specs/trust-store-trust-policy.md#trust-policy). | `[]` | | `notation.certs` | List of trusted root certificates for Notation verifier. | `[]` | diff --git a/deployments/ratify-gatekeeper-provider/templates/deployment.yaml b/deployments/ratify-gatekeeper-provider/templates/deployment.yaml index 50ace76c9..f6adde3a7 100644 --- a/deployments/ratify-gatekeeper-provider/templates/deployment.yaml +++ b/deployments/ratify-gatekeeper-provider/templates/deployment.yaml @@ -11,6 +11,12 @@ spec: {{- include "ratify.selectorLabels" . | nindent 6 }} template: metadata: + {{- if .Values.metrics.enabled }} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: {{ .Values.metrics.port | quote }} + prometheus.io/path: "/metrics" + {{- end }} labels: {{- include "ratify.selectorLabels" . | nindent 8 }} {{- if or (eq (index .Values.stores 0).credential.provider "azure") (eq (include "ratify.akvCertsProvided" .) "true") }} @@ -66,11 +72,20 @@ spec: {{- if (lookup "v1" "Secret" .Release.Namespace "gatekeeper-webhook-server-cert") }} - "--gatekeeper-ca-cert-file=/usr/local/tls/client-ca/ca.crt" {{- end }} + {{- if .Values.metrics.enabled }} + - "--enable-metrics" + - "--metrics-port={{ .Values.metrics.port | int }}" + {{- end }} ports: - containerPort: 6001 - containerPort: 9099 name: healthz protocol: TCP + {{- if .Values.metrics.enabled }} + - containerPort: {{ .Values.metrics.port | int }} + name: metrics + protocol: TCP + {{- end }} livenessProbe: httpGet: path: /healthz diff --git a/deployments/ratify-gatekeeper-provider/values.yaml b/deployments/ratify-gatekeeper-provider/values.yaml index a1aa7f2a8..02655c6de 100644 --- a/deployments/ratify-gatekeeper-provider/values.yaml +++ b/deployments/ratify-gatekeeper-provider/values.yaml @@ -18,6 +18,10 @@ resources: cpu: 600m memory: 512Mi +metrics: + enabled: true + port: 8888 + executor: scopes: [] concurrency: 3