Added metrics for lock acquiring attempts

Author: ivansanin

pkg/server/prometheus_exporter.go

@@ -1,41 +1,55 @@
 package server
 
 import (
-	"github.com/prometheus/client_golang/prometheus"
 	"strings"
+
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 var (
+	buildInfoGaugeVec = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "sidecache_admission_build_info",
+			Help: "Build info for sidecache admission webhook",
+		}, []string{"version"})
+
 	cacheHitCounter = prometheus.NewCounter(
 		prometheus.CounterOpts{
 			Namespace: "sidecache_" + ProjectName,
 			Name:      "cache_hit_counter",
 			Help:      "Cache hit count",
 		})
 
+	lockAcquiringAttemptsHistogram = prometheus.NewHistogram(
+		prometheus.HistogramOpts{
+			Namespace: "sidecache_" + ProjectName,
+			Name:      "lock_acquiring_attempts_histogram",
+			Help:      "Lock acquiring attempts histogram",
+			Buckets:   []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 100},
+		})
+
 	totalRequestCounter = prometheus.NewCounter(
 		prometheus.CounterOpts{
 			Namespace: "sidecache_" + ProjectName,
 			Name:      "all_request_hit_counter",
 			Help:      "All request hit counter",
 		})
-
-	buildInfoGaugeVec = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "sidecache_admission_build_info",
-			Help: "Build info for sidecache admission webhook",
-		}, []string{"version"})
 )
 
 type Prometheus struct {
-	CacheHitCounter     prometheus.Counter
-	TotalRequestCounter prometheus.Counter
+	CacheHitCounter                prometheus.Counter
+	LockAcquiringAttemptsHistogram prometheus.Histogram
+	TotalRequestCounter            prometheus.Counter
 }
 
 func NewPrometheusClient() *Prometheus {
-	prometheus.MustRegister(cacheHitCounter, totalRequestCounter, buildInfoGaugeVec)
+	prometheus.MustRegister(buildInfoGaugeVec, cacheHitCounter, lockAcquiringAttemptsHistogram, totalRequestCounter)
 
-	return &Prometheus{TotalRequestCounter: totalRequestCounter, CacheHitCounter: cacheHitCounter}
+	return &Prometheus{
+		CacheHitCounter:                cacheHitCounter,
+		LockAcquiringAttemptsHistogram: lockAcquiringAttemptsHistogram,
+		TotalRequestCounter:            totalRequestCounter,
+	}
 }
 
 func BuildInfo(admission string) {

pkg/server/server.go

@@ -7,7 +7,6 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"errors"
-	"github.com/zeriontech/sidecache/pkg/lock"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -19,6 +18,7 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/zeriontech/sidecache/pkg/cache"
+	"github.com/zeriontech/sidecache/pkg/lock"
 	"go.uber.org/zap"
 )
 
@@ -160,6 +160,7 @@ func (server CacheServer) CacheHandler(w http.ResponseWriter, r *http.Request) {
 			// try to acquire the lock
 			server.Logger.Info("acquiring the lock", zap.String("key", key))
 			if err := server.LockMgr.Acquire(key, LockTtl); err == nil {
+				server.Prometheus.LockAcquiringAttemptsHistogram.Observe(float64(attempt) + 1)
 				server.Logger.Info("lock acquired", zap.String("key", key))
 				defer func() {
 					// release the lock