From f1b6d83b94f70c8fe9b355e2000c4aab8de6e67b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szajbe?= Date: Tue, 23 Sep 2025 21:58:06 +0200 Subject: [PATCH 1/4] Make oban metrics buckets configurable --- lib/prom_ex/plugins/oban.ex | 37 +++++++++++++++++++++++------- test/prom_ex/plugins/oban_test.exs | 5 +++- test/support/metrics/oban.txt | 8 +++++++ 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/lib/prom_ex/plugins/oban.ex b/lib/prom_ex/plugins/oban.ex index 03218b4f..6b1a23d4 100644 --- a/lib/prom_ex/plugins/oban.ex +++ b/lib/prom_ex/plugins/oban.ex @@ -20,6 +20,14 @@ if Code.ensure_loaded?(Oban) do - `duration_unit`: This is an OPTIONAL option and is a `Telemetry.Metrics.time_unit()`. It can be one of: `:second | :millisecond | :microsecond | :nanosecond`. It is `:millisecond` by default. + - `job_attempt_buckets`: OPTIONAL. Buckets for job attempt distributions. Defaults to `[1, 5, 10]`. + + - `job_duration_buckets`: OPTIONAL. Buckets for job duration and queue time distributions + (in the configured `duration_unit`). Defaults to `[10, 100, 500, 1_000, 5_000, 20_000]`. + + - `producer_duration_buckets`: OPTIONAL. Buckets for producer duration distributions + (in the configured `duration_unit`). Defaults to `[10, 100, 500, 1_000, 5_000, 10_000]`. + This plugin exposes the following metric groups: - `:oban_init_event_metrics` - `:oban_job_event_metrics` @@ -74,6 +82,9 @@ if Code.ensure_loaded?(Oban) do otp_app = Keyword.fetch!(opts, :otp_app) metric_prefix = Keyword.get(opts, :metric_prefix, PromEx.metric_prefix(otp_app, :oban)) duration_unit = Keyword.get(opts, :duration_unit, :millisecond) + job_attempt_buckets = Keyword.get(opts, :job_attempt_buckets, [1, 5, 10]) + job_duration_buckets = Keyword.get(opts, :job_duration_buckets, [10, 100, 500, 1_000, 5_000, 20_000]) + producer_duration_buckets = Keyword.get(opts, :producer_duration_buckets, [10, 100, 500, 1_000, 5_000, 10_000]) oban_supervisors = get_oban_supervisors(opts) keep_function_filter = keep_oban_instance_metrics(oban_supervisors) @@ -83,8 +94,14 @@ if Code.ensure_loaded?(Oban) do [ oban_supervisor_init_event_metrics(metric_prefix, keep_function_filter, duration_unit), - oban_job_event_metrics(metric_prefix, keep_function_filter, duration_unit), - oban_producer_event_metrics(metric_prefix, keep_function_filter, duration_unit), + oban_job_event_metrics( + metric_prefix, + keep_function_filter, + duration_unit, + job_attempt_buckets, + job_duration_buckets + ), + oban_producer_event_metrics(metric_prefix, keep_function_filter, duration_unit, producer_duration_buckets), oban_circuit_breaker_event_metrics(metric_prefix, keep_function_filter) ] end @@ -186,9 +203,13 @@ if Code.ensure_loaded?(Oban) do } end - defp oban_job_event_metrics(metric_prefix, keep_function_filter, duration_unit) do - job_attempt_buckets = [1, 5, 10] - job_duration_buckets = [10, 100, 500, 1_000, 5_000, 20_000] + defp oban_job_event_metrics( + metric_prefix, + keep_function_filter, + duration_unit, + job_attempt_buckets, + job_duration_buckets + ) do duration_unit_plural = Utils.make_plural_atom(duration_unit) Event.build( @@ -279,7 +300,7 @@ if Code.ensure_loaded?(Oban) do ) end - defp oban_producer_event_metrics(metric_prefix, keep_function_filter, duration_unit) do + defp oban_producer_event_metrics(metric_prefix, keep_function_filter, duration_unit, producer_duration_buckets) do duration_unit_plural = Utils.make_plural_atom(duration_unit) Event.build( @@ -291,7 +312,7 @@ if Code.ensure_loaded?(Oban) do measurement: :duration, description: "How long it took to dispatch the job.", reporter_options: [ - buckets: [10, 100, 500, 1_000, 5_000, 10_000] + buckets: producer_duration_buckets ], unit: {:native, duration_unit}, tag_values: &producer_tag_values/1, @@ -318,7 +339,7 @@ if Code.ensure_loaded?(Oban) do measurement: :duration, description: "How long it took for the producer to raise an exception.", reporter_options: [ - buckets: [10, 100, 500, 1_000, 5_000, 10_000] + buckets: producer_duration_buckets ], unit: {:native, duration_unit}, tag_values: &producer_tag_values/1, diff --git a/test/prom_ex/plugins/oban_test.exs b/test/prom_ex/plugins/oban_test.exs index 267f3dd3..20af30a9 100644 --- a/test/prom_ex/plugins/oban_test.exs +++ b/test/prom_ex/plugins/oban_test.exs @@ -10,7 +10,10 @@ defmodule PromEx.Plugins.ObanTest do @impl true def plugins do - [{PromEx.Plugins.Oban, oban_supervisors: [Oban]}] + [ + {PromEx.Plugins.Oban, + oban_supervisors: [Oban], job_duration_buckets: [10, 100, 500, 1_000, 5_000, 10_000, 20_000]} + ] end end diff --git a/test/support/metrics/oban.txt b/test/support/metrics/oban.txt index 61bb3142..3be5e477 100644 --- a/test/support/metrics/oban.txt +++ b/test/support/metrics/oban.txt @@ -68,6 +68,7 @@ web_app_prom_ex_oban_job_exception_queue_time_milliseconds_bucket{error="Arithme web_app_prom_ex_oban_job_exception_queue_time_milliseconds_bucket{error="ArithmeticError",kind="error",name="Oban",queue="default",state="failure",worker="WebApp.Jobs.DefaultWorker",le="500"} 0 web_app_prom_ex_oban_job_exception_queue_time_milliseconds_bucket{error="ArithmeticError",kind="error",name="Oban",queue="default",state="failure",worker="WebApp.Jobs.DefaultWorker",le="1000"} 0 web_app_prom_ex_oban_job_exception_queue_time_milliseconds_bucket{error="ArithmeticError",kind="error",name="Oban",queue="default",state="failure",worker="WebApp.Jobs.DefaultWorker",le="5000"} 0 +web_app_prom_ex_oban_job_exception_queue_time_milliseconds_bucket{error="ArithmeticError",kind="error",name="Oban",queue="default",state="failure",worker="WebApp.Jobs.DefaultWorker",le="10000"} 0 web_app_prom_ex_oban_job_exception_queue_time_milliseconds_bucket{error="ArithmeticError",kind="error",name="Oban",queue="default",state="failure",worker="WebApp.Jobs.DefaultWorker",le="20000"} 0 web_app_prom_ex_oban_job_exception_queue_time_milliseconds_bucket{error="ArithmeticError",kind="error",name="Oban",queue="default",state="failure",worker="WebApp.Jobs.DefaultWorker",le="+Inf"} 3 web_app_prom_ex_oban_job_exception_queue_time_milliseconds_sum{error="ArithmeticError",kind="error",name="Oban",queue="default",state="failure",worker="WebApp.Jobs.DefaultWorker"} 110286.49199999998 @@ -79,6 +80,7 @@ web_app_prom_ex_oban_job_exception_duration_milliseconds_bucket{error="Arithmeti web_app_prom_ex_oban_job_exception_duration_milliseconds_bucket{error="ArithmeticError",kind="error",name="Oban",queue="default",state="failure",worker="WebApp.Jobs.DefaultWorker",le="500"} 3 web_app_prom_ex_oban_job_exception_duration_milliseconds_bucket{error="ArithmeticError",kind="error",name="Oban",queue="default",state="failure",worker="WebApp.Jobs.DefaultWorker",le="1000"} 3 web_app_prom_ex_oban_job_exception_duration_milliseconds_bucket{error="ArithmeticError",kind="error",name="Oban",queue="default",state="failure",worker="WebApp.Jobs.DefaultWorker",le="5000"} 3 +web_app_prom_ex_oban_job_exception_duration_milliseconds_bucket{error="ArithmeticError",kind="error",name="Oban",queue="default",state="failure",worker="WebApp.Jobs.DefaultWorker",le="10000"} 3 web_app_prom_ex_oban_job_exception_duration_milliseconds_bucket{error="ArithmeticError",kind="error",name="Oban",queue="default",state="failure",worker="WebApp.Jobs.DefaultWorker",le="20000"} 3 web_app_prom_ex_oban_job_exception_duration_milliseconds_bucket{error="ArithmeticError",kind="error",name="Oban",queue="default",state="failure",worker="WebApp.Jobs.DefaultWorker",le="+Inf"} 3 web_app_prom_ex_oban_job_exception_duration_milliseconds_sum{error="ArithmeticError",kind="error",name="Oban",queue="default",state="failure",worker="WebApp.Jobs.DefaultWorker"} 25.9197 @@ -110,6 +112,7 @@ web_app_prom_ex_oban_job_queue_time_milliseconds_bucket{name="Oban",queue="event web_app_prom_ex_oban_job_queue_time_milliseconds_bucket{name="Oban",queue="events",state="success",worker="WebApp.Jobs.EventWorker",le="500"} 1 web_app_prom_ex_oban_job_queue_time_milliseconds_bucket{name="Oban",queue="events",state="success",worker="WebApp.Jobs.EventWorker",le="1000"} 1 web_app_prom_ex_oban_job_queue_time_milliseconds_bucket{name="Oban",queue="events",state="success",worker="WebApp.Jobs.EventWorker",le="5000"} 1 +web_app_prom_ex_oban_job_queue_time_milliseconds_bucket{name="Oban",queue="events",state="success",worker="WebApp.Jobs.EventWorker",le="10000"} 1 web_app_prom_ex_oban_job_queue_time_milliseconds_bucket{name="Oban",queue="events",state="success",worker="WebApp.Jobs.EventWorker",le="20000"} 1 web_app_prom_ex_oban_job_queue_time_milliseconds_bucket{name="Oban",queue="events",state="success",worker="WebApp.Jobs.EventWorker",le="+Inf"} 1 web_app_prom_ex_oban_job_queue_time_milliseconds_sum{name="Oban",queue="events",state="success",worker="WebApp.Jobs.EventWorker"} 43.516 @@ -119,6 +122,7 @@ web_app_prom_ex_oban_job_queue_time_milliseconds_bucket{name="Oban",queue="defau web_app_prom_ex_oban_job_queue_time_milliseconds_bucket{name="Oban",queue="default",state="success",worker="WebApp.Jobs.DefaultWorker",le="500"} 0 web_app_prom_ex_oban_job_queue_time_milliseconds_bucket{name="Oban",queue="default",state="success",worker="WebApp.Jobs.DefaultWorker",le="1000"} 0 web_app_prom_ex_oban_job_queue_time_milliseconds_bucket{name="Oban",queue="default",state="success",worker="WebApp.Jobs.DefaultWorker",le="5000"} 0 +web_app_prom_ex_oban_job_queue_time_milliseconds_bucket{name="Oban",queue="default",state="success",worker="WebApp.Jobs.DefaultWorker",le="10000"} 0 web_app_prom_ex_oban_job_queue_time_milliseconds_bucket{name="Oban",queue="default",state="success",worker="WebApp.Jobs.DefaultWorker",le="20000"} 0 web_app_prom_ex_oban_job_queue_time_milliseconds_bucket{name="Oban",queue="default",state="success",worker="WebApp.Jobs.DefaultWorker",le="+Inf"} 8 web_app_prom_ex_oban_job_queue_time_milliseconds_sum{name="Oban",queue="default",state="success",worker="WebApp.Jobs.DefaultWorker"} 308053.534 @@ -128,6 +132,7 @@ web_app_prom_ex_oban_job_queue_time_milliseconds_bucket{name="Oban",queue="media web_app_prom_ex_oban_job_queue_time_milliseconds_bucket{name="Oban",queue="media",state="success",worker="WebApp.Jobs.MediaWorker",le="500"} 3 web_app_prom_ex_oban_job_queue_time_milliseconds_bucket{name="Oban",queue="media",state="success",worker="WebApp.Jobs.MediaWorker",le="1000"} 3 web_app_prom_ex_oban_job_queue_time_milliseconds_bucket{name="Oban",queue="media",state="success",worker="WebApp.Jobs.MediaWorker",le="5000"} 3 +web_app_prom_ex_oban_job_queue_time_milliseconds_bucket{name="Oban",queue="media",state="success",worker="WebApp.Jobs.MediaWorker",le="10000"} 3 web_app_prom_ex_oban_job_queue_time_milliseconds_bucket{name="Oban",queue="media",state="success",worker="WebApp.Jobs.MediaWorker",le="20000"} 3 web_app_prom_ex_oban_job_queue_time_milliseconds_bucket{name="Oban",queue="media",state="success",worker="WebApp.Jobs.MediaWorker",le="+Inf"} 3 web_app_prom_ex_oban_job_queue_time_milliseconds_sum{name="Oban",queue="media",state="success",worker="WebApp.Jobs.MediaWorker"} 137.731 @@ -139,6 +144,7 @@ web_app_prom_ex_oban_job_processing_duration_milliseconds_bucket{name="Oban",que web_app_prom_ex_oban_job_processing_duration_milliseconds_bucket{name="Oban",queue="default",state="success",worker="WebApp.Jobs.DefaultWorker",le="500"} 1 web_app_prom_ex_oban_job_processing_duration_milliseconds_bucket{name="Oban",queue="default",state="success",worker="WebApp.Jobs.DefaultWorker",le="1000"} 4 web_app_prom_ex_oban_job_processing_duration_milliseconds_bucket{name="Oban",queue="default",state="success",worker="WebApp.Jobs.DefaultWorker",le="5000"} 8 +web_app_prom_ex_oban_job_processing_duration_milliseconds_bucket{name="Oban",queue="default",state="success",worker="WebApp.Jobs.DefaultWorker",le="10000"} 8 web_app_prom_ex_oban_job_processing_duration_milliseconds_bucket{name="Oban",queue="default",state="success",worker="WebApp.Jobs.DefaultWorker",le="20000"} 8 web_app_prom_ex_oban_job_processing_duration_milliseconds_bucket{name="Oban",queue="default",state="success",worker="WebApp.Jobs.DefaultWorker",le="+Inf"} 8 web_app_prom_ex_oban_job_processing_duration_milliseconds_sum{name="Oban",queue="default",state="success",worker="WebApp.Jobs.DefaultWorker"} 8692.4147 @@ -148,6 +154,7 @@ web_app_prom_ex_oban_job_processing_duration_milliseconds_bucket{name="Oban",que web_app_prom_ex_oban_job_processing_duration_milliseconds_bucket{name="Oban",queue="events",state="success",worker="WebApp.Jobs.EventWorker",le="500"} 1 web_app_prom_ex_oban_job_processing_duration_milliseconds_bucket{name="Oban",queue="events",state="success",worker="WebApp.Jobs.EventWorker",le="1000"} 1 web_app_prom_ex_oban_job_processing_duration_milliseconds_bucket{name="Oban",queue="events",state="success",worker="WebApp.Jobs.EventWorker",le="5000"} 1 +web_app_prom_ex_oban_job_processing_duration_milliseconds_bucket{name="Oban",queue="events",state="success",worker="WebApp.Jobs.EventWorker",le="10000"} 1 web_app_prom_ex_oban_job_processing_duration_milliseconds_bucket{name="Oban",queue="events",state="success",worker="WebApp.Jobs.EventWorker",le="20000"} 1 web_app_prom_ex_oban_job_processing_duration_milliseconds_bucket{name="Oban",queue="events",state="success",worker="WebApp.Jobs.EventWorker",le="+Inf"} 1 web_app_prom_ex_oban_job_processing_duration_milliseconds_sum{name="Oban",queue="events",state="success",worker="WebApp.Jobs.EventWorker"} 108.77759999999999 @@ -157,6 +164,7 @@ web_app_prom_ex_oban_job_processing_duration_milliseconds_bucket{name="Oban",que web_app_prom_ex_oban_job_processing_duration_milliseconds_bucket{name="Oban",queue="media",state="success",worker="WebApp.Jobs.MediaWorker",le="500"} 3 web_app_prom_ex_oban_job_processing_duration_milliseconds_bucket{name="Oban",queue="media",state="success",worker="WebApp.Jobs.MediaWorker",le="1000"} 3 web_app_prom_ex_oban_job_processing_duration_milliseconds_bucket{name="Oban",queue="media",state="success",worker="WebApp.Jobs.MediaWorker",le="5000"} 3 +web_app_prom_ex_oban_job_processing_duration_milliseconds_bucket{name="Oban",queue="media",state="success",worker="WebApp.Jobs.MediaWorker",le="10000"} 3 web_app_prom_ex_oban_job_processing_duration_milliseconds_bucket{name="Oban",queue="media",state="success",worker="WebApp.Jobs.MediaWorker",le="20000"} 3 web_app_prom_ex_oban_job_processing_duration_milliseconds_bucket{name="Oban",queue="media",state="success",worker="WebApp.Jobs.MediaWorker",le="+Inf"} 3 web_app_prom_ex_oban_job_processing_duration_milliseconds_sum{name="Oban",queue="media",state="success",worker="WebApp.Jobs.MediaWorker"} 662.5332999999999 From 29ee12d726b8f63e3cd4ede9ebf420fdd567b67e Mon Sep 17 00:00:00 2001 From: Pedro Assumpcao Date: Tue, 9 Dec 2025 16:31:23 -0600 Subject: [PATCH 2/4] Use Oban instance config instead of default. --- lib/prom_ex/plugins/oban.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/prom_ex/plugins/oban.ex b/lib/prom_ex/plugins/oban.ex index 03218b4f..8907c586 100644 --- a/lib/prom_ex/plugins/oban.ex +++ b/lib/prom_ex/plugins/oban.ex @@ -438,7 +438,7 @@ if Code.ensure_loaded?(Oban) do config |> Oban.Repo.all(query) - |> include_zeros_for_missing_queue_states() + |> include_zeros_for_missing_queue_states(config) |> Enum.each(fn {{queue, state}, count} -> measurements = %{count: count} metadata = %{name: normalize_module_name(oban_supervisor), queue: queue, state: state} @@ -447,10 +447,10 @@ if Code.ensure_loaded?(Oban) do end) end - defp include_zeros_for_missing_queue_states(query_result) do + defp include_zeros_for_missing_queue_states(query_result, config) do {_, opts} = - Oban.config().plugins - |> Enum.find({nil, [queues: Oban.config().queues]}, fn {plugin, _} -> + config.plugins + |> Enum.find({nil, [queues: config.queues]}, fn {plugin, _} -> plugin == Oban.Pro.Plugins.DynamicQueues end) From ff5e3ffe1fb14199c0f1f9236903609b27af655a Mon Sep 17 00:00:00 2001 From: Pedro Assumpcao Date: Tue, 9 Dec 2025 16:44:37 -0600 Subject: [PATCH 3/4] Add tests for multiple Oban instances. --- test/prom_ex/plugins/oban_test.exs | 53 ++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/test/prom_ex/plugins/oban_test.exs b/test/prom_ex/plugins/oban_test.exs index 267f3dd3..fa9fe595 100644 --- a/test/prom_ex/plugins/oban_test.exs +++ b/test/prom_ex/plugins/oban_test.exs @@ -14,6 +14,15 @@ defmodule PromEx.Plugins.ObanTest do end end + defmodule MultiObanApp.PromEx do + use PromEx, otp_app: :multi_oban_app + + @impl true + def plugins do + [{PromEx.Plugins.Oban, oban_supervisors: [Oban, Oban.SuperSecret]}] + end + end + test "telemetry events are accumulated" do start_supervised!(WebApp.PromEx) @@ -22,6 +31,35 @@ defmodule PromEx.Plugins.ObanTest do Metrics.assert_prom_ex_metrics(WebApp.PromEx, :oban) end + test "telemetry events are accumulated for multiple Oban instances" do + start_supervised!(MultiObanApp.PromEx) + + Events.execute_all(:oban) + + collected_metrics = MultiObanApp.PromEx |> PromEx.get_metrics() |> String.split("\n", trim: true) + + assert Enum.any?(collected_metrics, fn line -> + String.contains?(line, ~s(name="Oban")) and String.contains?(line, ~s(queue="default")) + end) + + assert Enum.any?(collected_metrics, fn line -> + String.contains?(line, ~s(name="Oban.SuperSecret")) and + String.contains?(line, ~s(queue="default")) + end) + + assert Enum.any?(collected_metrics, fn line -> + String.contains?(line, ~s(name="Oban")) and + String.contains?(line, ~s(queue="events")) and + String.contains?(line, "25") + end) + + assert Enum.any?(collected_metrics, fn line -> + String.contains?(line, ~s(name="Oban.SuperSecret")) and + String.contains?(line, ~s(queue="events")) and + String.contains?(line, "50") + end) + end + describe "event_metrics/1" do test "should return the correct number of metrics" do assert [_, _, _, _] = ObanPlugin.event_metrics(otp_app: :prom_ex) @@ -32,6 +70,21 @@ defmodule PromEx.Plugins.ObanTest do test "should return the correct number of metrics" do assert %Polling{} = ObanPlugin.polling_metrics(otp_app: :prom_ex) end + + test "works with multiple Oban instances with different queue configurations" do + polling_metrics = + ObanPlugin.polling_metrics( + otp_app: :prom_ex, + oban_supervisors: [Oban, Oban.SuperSecret] + ) + + assert %Polling{} = polling_metrics + + assert {PromEx.Plugins.Oban, :execute_queue_metrics, [supervisors]} = + polling_metrics.measurements_mfa + + assert MapSet.equal?(MapSet.new(supervisors), MapSet.new([Oban, Oban.SuperSecret])) + end end describe "manual_metrics/1" do From cdc7688c95ca3c185f34a4db122dc48919b1f157 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szajbe?= Date: Wed, 11 Mar 2026 11:50:33 +0100 Subject: [PATCH 4/4] Allow to select all instances in broadway dashboard --- priv/broadway.json.eex | 53 +++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/priv/broadway.json.eex b/priv/broadway.json.eex index a6fa3305..48c95057 100644 --- a/priv/broadway.json.eex +++ b/priv/broadway.json.eex @@ -133,7 +133,7 @@ "targets": [ { "exemplar": true, - "expr": "<%= @broadway_metric_prefix %>_init_status_info{job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}", + "expr": "<%= @broadway_metric_prefix %>_init_status_info{job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}", "instant": true, "interval": "", "legendFormat": "", @@ -250,7 +250,7 @@ "targets": [ { "exemplar": true, - "expr": "{__name__=~\"<%= @broadway_metric_prefix %>_init_processor_concurrency_value|<%= @broadway_metric_prefix %>_init_processor_max_demand_value|<%= @broadway_metric_prefix %>_init_processor_hibernate_after_milliseconds\", job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}", + "expr": "{__name__=~\"<%= @broadway_metric_prefix %>_init_processor_concurrency_value|<%= @broadway_metric_prefix %>_init_processor_max_demand_value|<%= @broadway_metric_prefix %>_init_processor_hibernate_after_milliseconds\", job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}", "format": "table", "instant": true, "interval": "", @@ -388,7 +388,7 @@ "targets": [ { "exemplar": true, - "expr": "{__name__=~\"<%= @broadway_metric_prefix %>_init_batcher_batch_timeout_milliseconds|<%= @broadway_metric_prefix %>_init_batcher_batch_size_value|<%= @broadway_metric_prefix %>_init_batcher_concurrency_value|<%= @broadway_metric_prefix %>_init_batcher_hibernate_after_milliseconds\", job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}", + "expr": "{__name__=~\"<%= @broadway_metric_prefix %>_init_batcher_batch_timeout_milliseconds|<%= @broadway_metric_prefix %>_init_batcher_batch_size_value|<%= @broadway_metric_prefix %>_init_batcher_concurrency_value|<%= @broadway_metric_prefix %>_init_batcher_hibernate_after_milliseconds\", job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}", "format": "table", "instant": true, "interval": "", @@ -517,7 +517,7 @@ "targets": [ { "exemplar": true, - "expr": "{__name__=~\"<%= @broadway_metric_prefix %>_init_shutdown_default_milliseconds|<%= @broadway_metric_prefix %>_init_max_restarts_default_value|<%= @broadway_metric_prefix %>_init_max_duration_default_milliseconds|<%= @broadway_metric_prefix %>_init_resubscribe_interval_default_milliseconds|<%= @broadway_metric_prefix %>_init_hibernate_after_default_milliseconds\", job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}", + "expr": "{__name__=~\"<%= @broadway_metric_prefix %>_init_shutdown_default_milliseconds|<%= @broadway_metric_prefix %>_init_max_restarts_default_value|<%= @broadway_metric_prefix %>_init_max_duration_default_milliseconds|<%= @broadway_metric_prefix %>_init_resubscribe_interval_default_milliseconds|<%= @broadway_metric_prefix %>_init_hibernate_after_default_milliseconds\", job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}", "format": "table", "instant": true, "interval": "", @@ -606,7 +606,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(increase(<%= @broadway_metric_prefix %>_process_message_duration_milliseconds_count{job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}[24h])) / sum(increase({__name__=~\"<%= @broadway_metric_prefix %>_process_message_duration_milliseconds_count|<%= @broadway_metric_prefix %>_process_message_exception_duration_milliseconds_count\", job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}[24h])) OR on() vector(0)", + "expr": "sum(increase(<%= @broadway_metric_prefix %>_process_message_duration_milliseconds_count{job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}[24h])) / sum(increase({__name__=~\"<%= @broadway_metric_prefix %>_process_message_duration_milliseconds_count|<%= @broadway_metric_prefix %>_process_message_exception_duration_milliseconds_count\", job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}[24h])) OR on() vector(0)", "instant": false, "interval": "", "legendFormat": "", @@ -664,7 +664,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(increase(<%= @broadway_metric_prefix %>_process_message_duration_milliseconds_count{job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}[24h]))", + "expr": "sum(increase(<%= @broadway_metric_prefix %>_process_message_duration_milliseconds_count{job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}[24h]))", "instant": false, "interval": "", "legendFormat": "", @@ -722,7 +722,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(increase(<%= @broadway_metric_prefix %>_process_batch_duration_milliseconds_count{job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}[24h]))", + "expr": "sum(increase(<%= @broadway_metric_prefix %>_process_batch_duration_milliseconds_count{job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}[24h]))", "instant": false, "interval": "", "legendFormat": "", @@ -793,7 +793,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(increase(<%= @broadway_metric_prefix %>_process_message_duration_milliseconds_count{job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}[1h])) / sum(increase({__name__=~\"<%= @broadway_metric_prefix %>_process_message_duration_milliseconds_count|<%= @broadway_metric_prefix %>_process_message_exception_duration_milliseconds_count\", job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}[1h])) OR on() vector(0)", + "expr": "sum(increase(<%= @broadway_metric_prefix %>_process_message_duration_milliseconds_count{job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}[1h])) / sum(increase({__name__=~\"<%= @broadway_metric_prefix %>_process_message_duration_milliseconds_count|<%= @broadway_metric_prefix %>_process_message_exception_duration_milliseconds_count\", job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}[1h])) OR on() vector(0)", "instant": false, "interval": "", "legendFormat": "", @@ -851,7 +851,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(increase(<%= @broadway_metric_prefix %>_process_message_duration_milliseconds_count{job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}[1h]))", + "expr": "sum(increase(<%= @broadway_metric_prefix %>_process_message_duration_milliseconds_count{job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}[1h]))", "instant": false, "interval": "", "legendFormat": "", @@ -909,7 +909,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(increase(<%= @broadway_metric_prefix %>_process_batch_duration_milliseconds_count{job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}[1h]))", + "expr": "sum(increase(<%= @broadway_metric_prefix %>_process_batch_duration_milliseconds_count{job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}[1h]))", "instant": false, "interval": "", "legendFormat": "", @@ -968,7 +968,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(irate(<%= @broadway_metric_prefix %>_process_message_duration_milliseconds_bucket{job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}[$interval])) by (le)", + "expr": "sum(irate(<%= @broadway_metric_prefix %>_process_message_duration_milliseconds_bucket{job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}[$interval])) by (le)", "format": "heatmap", "hide": false, "interval": "", @@ -1035,7 +1035,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(irate(<%= @broadway_metric_prefix %>_process_message_exception_duration_milliseconds_bucket{job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}[$interval])) by (le)", + "expr": "sum(irate(<%= @broadway_metric_prefix %>_process_message_exception_duration_milliseconds_bucket{job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}[$interval])) by (le)", "format": "heatmap", "hide": false, "interval": "", @@ -1117,7 +1117,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(irate(<%= @broadway_metric_prefix %>_process_message_duration_milliseconds_sum{job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}[$interval])) by(processor_key) / sum(irate(<%= @broadway_metric_prefix %>_process_message_duration_milliseconds_count{job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}[$interval])) by(processor_key)", + "expr": "sum(irate(<%= @broadway_metric_prefix %>_process_message_duration_milliseconds_sum{job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}[$interval])) by(processor_key) / sum(irate(<%= @broadway_metric_prefix %>_process_message_duration_milliseconds_count{job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}[$interval])) by(processor_key)", "interval": "", "legendFormat": "{{ processor_key }}", "refId": "A" @@ -1212,7 +1212,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(irate(<%= @broadway_metric_prefix %>_process_message_exception_duration_milliseconds_sum{job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}[$interval])) by(processor_key, reason) / sum(irate(<%= @broadway_metric_prefix %>_process_message_exception_duration_milliseconds_count{job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}[$interval])) by(processor_key, reason)", + "expr": "sum(irate(<%= @broadway_metric_prefix %>_process_message_exception_duration_milliseconds_sum{job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}[$interval])) by(processor_key, reason) / sum(irate(<%= @broadway_metric_prefix %>_process_message_exception_duration_milliseconds_count{job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}[$interval])) by(processor_key, reason)", "interval": "", "legendFormat": "{{ processor_key }} :: {{ reason }}", "refId": "A" @@ -1307,7 +1307,7 @@ "targets": [ { "exemplar": true, - "expr": "irate(<%= @broadway_metric_prefix %>_process_message_duration_milliseconds_count{job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}[$interval])", + "expr": "irate(<%= @broadway_metric_prefix %>_process_message_duration_milliseconds_count{job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}[$interval])", "interval": "", "legendFormat": "{{ processor_key }}", "refId": "A" @@ -1402,7 +1402,7 @@ "targets": [ { "exemplar": true, - "expr": "irate(<%= @broadway_metric_prefix %>_process_message_exception_duration_milliseconds_count{job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}[$interval])", + "expr": "irate(<%= @broadway_metric_prefix %>_process_message_exception_duration_milliseconds_count{job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}[$interval])", "interval": "", "legendFormat": "{{ processor_key }} :: {{ reason }}", "refId": "A" @@ -1496,7 +1496,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(irate(<%= @broadway_metric_prefix %>_process_batch_duration_milliseconds_bucket{job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}[$interval])) by (le)", + "expr": "sum(irate(<%= @broadway_metric_prefix %>_process_batch_duration_milliseconds_bucket{job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}[$interval])) by (le)", "format": "heatmap", "hide": false, "interval": "", @@ -1578,7 +1578,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(irate(<%= @broadway_metric_prefix %>_process_batch_success_size_sum{job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}[$interval])) by(batcher) / sum(irate(<%= @broadway_metric_prefix %>_process_batch_success_size_count{job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}[$interval])) by(batcher)", + "expr": "sum(irate(<%= @broadway_metric_prefix %>_process_batch_success_size_sum{job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}[$interval])) by(batcher) / sum(irate(<%= @broadway_metric_prefix %>_process_batch_success_size_count{job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}[$interval])) by(batcher)", "interval": "", "legendFormat": "{{ batcher }}", "refId": "A" @@ -1673,7 +1673,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(irate(<%= @broadway_metric_prefix %>_process_batch_duration_milliseconds_sum{job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}[$interval])) by(batcher) / sum(irate(<%= @broadway_metric_prefix %>_process_batch_duration_milliseconds_count{job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}[$interval])) by(batcher)", + "expr": "sum(irate(<%= @broadway_metric_prefix %>_process_batch_duration_milliseconds_sum{job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}[$interval])) by(batcher) / sum(irate(<%= @broadway_metric_prefix %>_process_batch_duration_milliseconds_count{job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}[$interval])) by(batcher)", "interval": "", "legendFormat": "{{ batcher }}", "refId": "A" @@ -1768,7 +1768,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(irate(<%= @broadway_metric_prefix %>_process_batch_failure_size_sum{job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}[$interval])) by(batcher) / sum(irate(<%= @broadway_metric_prefix %>_process_batch_failure_size_count{job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}[$interval])) by(batcher)", + "expr": "sum(irate(<%= @broadway_metric_prefix %>_process_batch_failure_size_sum{job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}[$interval])) by(batcher) / sum(irate(<%= @broadway_metric_prefix %>_process_batch_failure_size_count{job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}[$interval])) by(batcher)", "interval": "", "legendFormat": "{{ batcher }}", "refId": "A" @@ -1863,7 +1863,7 @@ "targets": [ { "exemplar": true, - "expr": "irate(<%= @broadway_metric_prefix %>_process_batch_duration_milliseconds_count{job=\"$job\", instance=\"$instance\", name=\"$broadway_module\"}[$interval])", + "expr": "irate(<%= @broadway_metric_prefix %>_process_batch_duration_milliseconds_count{job=\"$job\", instance=~\"$instance\", name=\"$broadway_module\"}[$interval])", "interval": "", "legendFormat": "{{ batcher }}", "refId": "A" @@ -1940,15 +1940,20 @@ "useTags": false }, { - "allValue": null, + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, "datasource": "<%= @datasource_id %>", "definition": "label_values(<%= @otp_app %>_prom_ex_prom_ex_status_info, instance)", "description": null, "error": null, "hide": 0, - "includeAll": false, + "includeAll": true, "label": "Application Instance", - "multi": false, + "multi": true, "name": "instance", "options": [], "query": "label_values(<%= @otp_app %>_prom_ex_prom_ex_status_info{job=\"$job\"}, instance)",