From f9a34142143bea84ee409d7361e9c45092cac3b0 Mon Sep 17 00:00:00 2001 From: Alexander Diana Date: Mon, 13 Jan 2025 22:25:29 +0000 Subject: [PATCH 1/4] add prometheus metrics for summary queue info on requests for outdated summaries --- data/summary/store/store.go | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/data/summary/store/store.go b/data/summary/store/store.go index e777d9be58..95508e6918 100644 --- a/data/summary/store/store.go +++ b/data/summary/store/store.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "time" "go.mongodb.org/mongo-driver/bson" @@ -15,6 +17,17 @@ import ( storeStructuredMongo "github.com/tidepool-org/platform/store/structured/mongo" ) +var ( + QueueLag = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "tidepool_summary_queue_lag", + Help: "The current queue lag in minutes", + }, []string{"type"}) + QueueLength = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "tidepool_summary_queue_length", + Help: "The current queue length in number of summaries", + }, []string{"type"}) +) + type Repo[A types.StatsPt[T], T types.Stats] struct { *storeStructuredMongo.Repository } @@ -213,6 +226,14 @@ func (r *Repo[A, T]) SetOutdated(ctx context.Context, userId, reason string) (*t return userSummary.Dates.OutdatedSince, nil } +func (r *Repo[T, A]) GetSummaryQueueLength(ctx context.Context) (int64, error) { + selector := bson.M{ + "type": types.GetTypeString[T, A](), + "dates.outdatedSince": bson.M{"$lte": time.Now().UTC()}, + } + return r.CountDocuments(ctx, selector) +} + func (r *Repo[T, A]) GetOutdatedUserIDs(ctx context.Context, page *page.Pagination) (*types.OutdatedSummariesResponse, error) { if ctx == nil { return nil, errors.New("context is missing") @@ -221,8 +242,10 @@ func (r *Repo[T, A]) GetOutdatedUserIDs(ctx context.Context, page *page.Paginati return nil, errors.New("pagination is missing") } + typ := types.GetTypeString[T, A]() + selector := bson.M{ - "type": types.GetTypeString[T, A](), + "type": typ, "dates.outdatedSince": bson.M{"$lte": time.Now().UTC()}, } @@ -231,7 +254,7 @@ func (r *Repo[T, A]) GetOutdatedUserIDs(ctx context.Context, page *page.Paginati {Key: "dates.outdatedSince", Value: 1}, }) opts.SetLimit(int64(page.Size)) - opts.SetProjection(bson.M{"stats": 0}) + opts.SetProjection(bson.M{"userId": 1, "dates": 1}) cursor, err := r.Find(ctx, selector, opts) if err != nil { @@ -260,6 +283,13 @@ func (r *Repo[T, A]) GetOutdatedUserIDs(ctx context.Context, page *page.Paginati response.End = *userSummary.Dates.OutdatedSince } + QueueLag.WithLabelValues(typ).Observe(time.Now().UTC().Sub(response.Start).Minutes()) + count, err := r.GetSummaryQueueLength(ctx) + if err != nil { + return nil, fmt.Errorf("unable to get summary queue length: %w", err) + } + QueueLength.WithLabelValues(typ).Set(float64(count)) + return response, nil } From 0773c2882d13ce7bcf9e132775d45c66da225573 Mon Sep 17 00:00:00 2001 From: Alexander Diana Date: Mon, 13 Jan 2025 22:33:58 +0000 Subject: [PATCH 2/4] formatting --- data/summary/store/store.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data/summary/store/store.go b/data/summary/store/store.go index 95508e6918..ce99a2a068 100644 --- a/data/summary/store/store.go +++ b/data/summary/store/store.go @@ -4,9 +4,10 @@ import ( "context" "errors" "fmt" + "time" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" - "time" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" From 29acc449177d5d748d99164b337ddae1c84e97ec Mon Sep 17 00:00:00 2001 From: Alexander Diana Date: Thu, 16 Jan 2025 17:34:47 +0000 Subject: [PATCH 3/4] change metric to seconds instead of minutes, customize buckets, add metrics route --- data/service/api/v1/v1.go | 14 ++++++++++++++ data/summary/store/store.go | 7 ++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/data/service/api/v1/v1.go b/data/service/api/v1/v1.go index 9cf17c9517..446d55da4c 100644 --- a/data/service/api/v1/v1.go +++ b/data/service/api/v1/v1.go @@ -1,10 +1,23 @@ package v1 import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/tidepool-org/platform/data/service" + dataService "github.com/tidepool-org/platform/data/service" "github.com/tidepool-org/platform/service/api" + "net/http" ) +func PrometheusMetrics(dataServiceContext dataService.Context) { + res := dataServiceContext.Response() + req := dataServiceContext.Request() + + // The default go-json-rest middleware gzips the content + promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{DisableCompression: true}). + ServeHTTP(res.(http.ResponseWriter), req.Request) +} + func Routes() []service.Route { routes := []service.Route{ service.Post("/v1/datasets/:dataSetId/data", DataSetsDataCreate, api.RequireAuth), @@ -20,6 +33,7 @@ func Routes() []service.Route { service.Put("/v1/data_sets/:dataSetId", DataSetsUpdate, api.RequireAuth), service.Get("/v1/time", TimeGet), service.Post("/v1/users/:userId/data_sets", UsersDataSetsCreate, api.RequireAuth), + service.Get("/v1/metrics", PrometheusMetrics), } routes = append(routes, DataSetsRoutes()...) diff --git a/data/summary/store/store.go b/data/summary/store/store.go index ce99a2a068..14c98eab1b 100644 --- a/data/summary/store/store.go +++ b/data/summary/store/store.go @@ -20,8 +20,9 @@ import ( var ( QueueLag = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "tidepool_summary_queue_lag", - Help: "The current queue lag in minutes", + Name: "tidepool_summary_queue_lag", + Help: "The current queue lag in seconds", + Buckets: []float64{0.5, 1, 2.5, 5, 7.5, 10, 25, 50, 75, 100, 150, 250, 500, 1000}, }, []string{"type"}) QueueLength = promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "tidepool_summary_queue_length", @@ -284,7 +285,7 @@ func (r *Repo[T, A]) GetOutdatedUserIDs(ctx context.Context, page *page.Paginati response.End = *userSummary.Dates.OutdatedSince } - QueueLag.WithLabelValues(typ).Observe(time.Now().UTC().Sub(response.Start).Minutes()) + QueueLag.WithLabelValues(typ).Observe(time.Now().UTC().Sub(response.Start).Seconds()) count, err := r.GetSummaryQueueLength(ctx) if err != nil { return nil, fmt.Errorf("unable to get summary queue length: %w", err) From 2d8849af94bb6962909dbea5983c5ce78f027d68 Mon Sep 17 00:00:00 2001 From: Alexander Diana Date: Thu, 16 Jan 2025 17:35:42 +0000 Subject: [PATCH 4/4] formatting --- data/service/api/v1/v1.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/data/service/api/v1/v1.go b/data/service/api/v1/v1.go index 446d55da4c..abb6d14fba 100644 --- a/data/service/api/v1/v1.go +++ b/data/service/api/v1/v1.go @@ -1,12 +1,14 @@ package v1 import ( + "net/http" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/tidepool-org/platform/data/service" dataService "github.com/tidepool-org/platform/data/service" "github.com/tidepool-org/platform/service/api" - "net/http" ) func PrometheusMetrics(dataServiceContext dataService.Context) {