Skip to content

Commit

Permalink
upgrade opentelemtry instrumentation (#265)
Browse files Browse the repository at this point in the history
  • Loading branch information
st3veV authored Jul 20, 2023
1 parent 83b2914 commit 47d308d
Show file tree
Hide file tree
Showing 4 changed files with 58 additions and 65 deletions.
20 changes: 10 additions & 10 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,19 +9,19 @@ require (
github.com/golang/mock v1.6.0
github.com/kelseyhightower/envconfig v1.4.0
github.com/minio/minio-go/v7 v7.0.22
github.com/prometheus/client_golang v1.13.0
github.com/prometheus/client_golang v1.14.0
github.com/rs/cors v1.8.2
github.com/spf13/afero v1.9.3
github.com/spf13/cobra v1.5.0
github.com/stretchr/testify v1.8.1
github.com/stretchr/testify v1.8.2
github.com/technosophos/moniker v0.0.0-20210218184952-3ea787d3943b
github.com/tidwall/sjson v1.2.5
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.37.0
go.opentelemetry.io/otel v1.11.2
go.opentelemetry.io/otel/exporters/prometheus v0.33.0
go.opentelemetry.io/otel/metric v0.34.0
go.opentelemetry.io/otel/sdk v1.11.2
go.opentelemetry.io/otel/sdk/metric v0.34.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.40.0
go.opentelemetry.io/otel v1.14.0
go.opentelemetry.io/otel/exporters/prometheus v0.37.0
go.opentelemetry.io/otel/metric v0.37.0
go.opentelemetry.io/otel/sdk v1.14.0
go.opentelemetry.io/otel/sdk/metric v0.37.0
go.uber.org/zap v1.23.0
k8s.io/api v0.25.1
k8s.io/apimachinery v0.25.1
Expand Down Expand Up @@ -67,7 +67,7 @@ require (
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/rs/xid v1.2.1 // indirect
Expand All @@ -76,7 +76,7 @@ require (
github.com/tidwall/gjson v1.14.3 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
go.opentelemetry.io/otel/trace v1.11.2 // indirect
go.opentelemetry.io/otel/trace v1.14.0 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
golang.org/x/crypto v0.0.0-20221010152910-d6f0a8c073c2 // indirect
Expand Down
39 changes: 20 additions & 19 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -277,13 +277,14 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU=
github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ=
github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw=
github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
Expand Down Expand Up @@ -328,8 +329,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/technosophos/moniker v0.0.0-20210218184952-3ea787d3943b h1:fo0GUa0B+vxSZ8bgnL3fpCPHReM/QPlALdak9T/Zw5Y=
github.com/technosophos/moniker v0.0.0-20210218184952-3ea787d3943b/go.mod h1:O1c8HleITsZqzNZDjSNzirUGsMT0oGu9LhHKoJrqO+A=
github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
Expand All @@ -356,20 +357,20 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.37.0 h1:yt2NKzK7Vyo6h0+X8BA4FpreZQTlVEIarnsBP/H5mzs=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.37.0/go.mod h1:+ARmXlUlc51J7sZeCBkBJNdHGySrdOzgzxp6VWRWM1U=
go.opentelemetry.io/otel v1.11.2 h1:YBZcQlsVekzFsFbjygXMOXSs6pialIZxcjfO/mBDmR0=
go.opentelemetry.io/otel v1.11.2/go.mod h1:7p4EUV+AqgdlNV9gL97IgUZiVR3yrFXYo53f9BM3tRI=
go.opentelemetry.io/otel/exporters/prometheus v0.33.0 h1:xXhPj7SLKWU5/Zd4Hxmd+X1C4jdmvc0Xy+kvjFx2z60=
go.opentelemetry.io/otel/exporters/prometheus v0.33.0/go.mod h1:ZSmYfKdYWEdSDBB4njLBIwTf4AU2JNsH3n2quVQDebI=
go.opentelemetry.io/otel/metric v0.34.0 h1:MCPoQxcg/26EuuJwpYN1mZTeCYAUGx8ABxfW07YkjP8=
go.opentelemetry.io/otel/metric v0.34.0/go.mod h1:ZFuI4yQGNCupurTXCwkeD/zHBt+C2bR7bw5JqUm/AP8=
go.opentelemetry.io/otel/sdk v1.11.2 h1:GF4JoaEx7iihdMFu30sOyRx52HDHOkl9xQ8SMqNXUiU=
go.opentelemetry.io/otel/sdk v1.11.2/go.mod h1:wZ1WxImwpq+lVRo4vsmSOxdd+xwoUJ6rqyLc3SyX9aU=
go.opentelemetry.io/otel/sdk/metric v0.34.0 h1:7ElxfQpXCFZlRTvVRTkcUvK8Gt5DC8QzmzsLsO2gdzo=
go.opentelemetry.io/otel/sdk/metric v0.34.0/go.mod h1:l4r16BIqiqPy5rd14kkxllPy/fOI4tWo1jkpD9Z3ffQ=
go.opentelemetry.io/otel/trace v1.11.2 h1:Xf7hWSF2Glv0DE3MH7fBHvtpSBsjcBUe5MYAmZM/+y0=
go.opentelemetry.io/otel/trace v1.11.2/go.mod h1:4N+yC7QEz7TTsG9BSRLNAa63eg5E06ObSbKPmxQ/pKA=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.40.0 h1:lE9EJyw3/JhrjWH/hEy9FptnalDQgj7vpbgC2KCCCxE=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.40.0/go.mod h1:pcQ3MM3SWvrA71U4GDqv9UFDJ3HQsW7y5ZO3tDTlUdI=
go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM=
go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU=
go.opentelemetry.io/otel/exporters/prometheus v0.37.0 h1:NQc0epfL0xItsmGgSXgfbH2C1fq2VLXkZoDFsfRNHpc=
go.opentelemetry.io/otel/exporters/prometheus v0.37.0/go.mod h1:hB8qWjsStK36t50/R0V2ULFb4u95X/Q6zupXLgvjTh8=
go.opentelemetry.io/otel/metric v0.37.0 h1:pHDQuLQOZwYD+Km0eb657A25NaRzy0a+eLyKfDXedEs=
go.opentelemetry.io/otel/metric v0.37.0/go.mod h1:DmdaHfGt54iV6UKxsV9slj2bBRJcKC1B1uvDLIioc1s=
go.opentelemetry.io/otel/sdk v1.14.0 h1:PDCppFRDq8A1jL9v6KMI6dYesaq+DFcDZvjsoGvxGzY=
go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM=
go.opentelemetry.io/otel/sdk/metric v0.37.0 h1:haYBBtZZxiI3ROwSmkZnI+d0+AVzBWeviuYQDeBWosU=
go.opentelemetry.io/otel/sdk/metric v0.37.0/go.mod h1:mO2WV1AZKKwhwHTV3AKOoIEb9LbUaENZDuGUQd+j4A0=
go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M=
go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
Expand Down
23 changes: 9 additions & 14 deletions pkg/controller/loadtest.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@ import (
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/metric/instrument"
"go.opentelemetry.io/otel/metric/instrument/syncint64"
"go.opentelemetry.io/otel/metric/unit"

"go.uber.org/zap"
batchV1 "k8s.io/api/batch/v1"
Expand Down Expand Up @@ -43,37 +41,35 @@ const (

// MetricsReporter used to interface with the metrics configurations
type MetricsReporter struct {
workQueueDepthStat syncint64.UpDownCounter
reconcileCountStat syncint64.UpDownCounter
reconcileLatencyStat syncint64.Histogram
workQueueDepthStat instrument.Int64UpDownCounter
reconcileCountStat instrument.Int64UpDownCounter
reconcileLatencyStat instrument.Int64Histogram
}

// NewMetricsReporter contains loadtest metrics definition
func NewMetricsReporter(meter metric.Meter) (*MetricsReporter, error) {
workQueueDepthStat, err := meter.SyncInt64().UpDownCounter(
workQueueDepthStat, err := meter.Int64UpDownCounter(
"kangal_work_queue_depth",
instrument.WithDescription("Depth of the work queue"),
instrument.WithUnit(unit.Dimensionless),
)
if err != nil {
fmt.Errorf("could not register workQueueDepthStat metric: %w", err)
return nil, err
}

reconcileCountStat, err := meter.SyncInt64().UpDownCounter(
reconcileCountStat, err := meter.Int64UpDownCounter(
"kangal_reconcile_count",
instrument.WithDescription("Number of reconcile operations"),
instrument.WithUnit(unit.Dimensionless),
)
if err != nil {
fmt.Errorf("could not register reconcileCountStat metric: %w", err)
return nil, err
}

reconcileLatencyStat, err := meter.SyncInt64().Histogram(
reconcileLatencyStat, err := meter.Int64Histogram(
"kangal_reconcile_latency",
instrument.WithDescription("Latency of reconcile operations"),
instrument.WithUnit(unit.Milliseconds),
instrument.WithUnit("ms"),
)
if err != nil {
fmt.Errorf("could not register reconcileLatencyStat metric: %w", err)
Expand Down Expand Up @@ -127,8 +123,8 @@ func NewController(
kangalInformerFactory externalversions.SharedInformerFactory,
statsClient MetricsReporter,
registry backends.Registry,
logger *zap.Logger) *Controller {

logger *zap.Logger,
) *Controller {
namespaceInformer := kubeInformerFactory.Core().V1().Namespaces()
podInformer := kubeInformerFactory.Core().V1().Pods()
jobInformer := kubeInformerFactory.Batch().V1().Jobs()
Expand Down Expand Up @@ -313,7 +309,6 @@ func (c *Controller) processNextWorkItem() bool {
c.logger.Debug("Successfully synced", zap.String("loadtest", key))
return nil
}(obj)

if err != nil {
utilRuntime.HandleError(err)
return true
Expand Down
41 changes: 19 additions & 22 deletions pkg/proxy/proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@ import (
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/metric/instrument"
"go.opentelemetry.io/otel/metric/instrument/asyncint64"
"go.opentelemetry.io/otel/metric/unit"

"github.com/go-chi/chi/v5"
"github.com/go-chi/render"
Expand Down Expand Up @@ -41,38 +39,37 @@ type Proxy struct {

// MetricsReporter used to interface with the metrics configurations
type MetricsReporter struct {
countRunningLoadtests asyncint64.UpDownCounter
countRunningLoadtests instrument.Int64ObservableUpDownCounter
}

// NewMetricsReporter contains loadtest metrics definition
func NewMetricsReporter(meter metric.Meter, kubeClient *kube.Client) (*MetricsReporter, error) {
countRunningLoadtests, err := meter.AsyncInt64().UpDownCounter(
countRunningLoadtests, err := meter.Int64ObservableUpDownCounter(
"kangal_loadtests_count",
instrument.WithDescription("Current number of loadtests in cluster, grouped by type and phase"),
instrument.WithUnit(unit.Dimensionless),
instrument.WithInt64Callback(func(ctx context.Context, io instrument.Int64Observer) error {
states, types, err := kubeClient.CountExistingLoadtests()
if err != nil {
fmt.Errorf("could not get metric data for CountExistingLoadtests: %w", err)
return err
}
for k, v := range states {
io.Observe(v, attribute.String("phase", k.String()))
}

for k, v := range types {
io.Observe(v, attribute.String("type", k.String()))
}
return nil
}),
)
if err != nil {
return nil, fmt.Errorf("could not register countRunningLoadtests metric: %w", err)
}

if err := meter.RegisterCallback([]instrument.Asynchronous{countRunningLoadtests}, func(ctx context.Context) {
states, types, err := kubeClient.CountExistingLoadtests()
if err != nil {
fmt.Errorf("could not get metric data for CountExistingLoadtests: %w", err)
}
for k, v := range states {
countRunningLoadtests.Observe(ctx, v, attribute.String("phase", k.String()))
}

for k, v := range types {
countRunningLoadtests.Observe(ctx, v, attribute.String("type", k.String()))
}
},
); err != nil {
return nil, err
}
return &MetricsReporter{
countRunningLoadtests: countRunningLoadtests},
countRunningLoadtests: countRunningLoadtests,
},
nil
}

Expand Down

0 comments on commit 47d308d

Please sign in to comment.