From f6d052b2bf301657a298bd091d2d949e4331c5eb Mon Sep 17 00:00:00 2001 From: Brieuc Tribouillet Date: Fri, 27 Feb 2026 07:38:31 +0800 Subject: [PATCH 1/3] Fix ETSCronFlusher crash on flush timeout Replace Task.await with Task.yield + Task.shutdown so the GenServer logs a warning instead of crashing when ETS flush exceeds the 10s timeout. Fixes #279. --- lib/prom_ex/ets_cron_flusher.ex | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/prom_ex/ets_cron_flusher.ex b/lib/prom_ex/ets_cron_flusher.ex index bd8e91da..64cba7c9 100644 --- a/lib/prom_ex/ets_cron_flusher.ex +++ b/lib/prom_ex/ets_cron_flusher.ex @@ -8,6 +8,8 @@ defmodule PromEx.ETSCronFlusher do use GenServer + require Logger + @flush_timeout 10_000 @doc """ @@ -58,7 +60,13 @@ defmodule PromEx.ETSCronFlusher do PromEx.get_metrics(state.prom_ex_module) end) - Task.await(flush_task, @flush_timeout) + case Task.yield(flush_task, @flush_timeout) || Task.shutdown(flush_task) do + {:ok, _result} -> + :ok + + nil -> + Logger.warning("[PromEx.ETSCronFlusher] ETS flush timed out after #{@flush_timeout}ms") + end timer_ref = schedule_flush(state) {:noreply, %{state | timer_ref: timer_ref}} From 6e3e9d460c64393890c83d2380d63bd19193aa92 Mon Sep 17 00:00:00 2001 From: Brieuc Tribouillet Date: Fri, 27 Feb 2026 07:50:27 +0800 Subject: [PATCH 2/3] Fix ETSCronFlusher crash on flush timeout Replace Task.await with Task.yield + Task.shutdown so the GenServer logs a warning instead of crashing when ETS flush exceeds the timeout. Also make the flush timeout configurable via `ets_flush_timeout` config option (default: 10_000ms), following the same pattern as `ets_flush_interval`. Fixes #279 --- lib/prom_ex.ex | 10 +++++++--- lib/prom_ex/config.ex | 8 ++++++++ lib/prom_ex/ets_cron_flusher.ex | 11 ++++------- test/prom_ex/ets_cron_flusher_test.exs | 1 + 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/lib/prom_ex.ex b/lib/prom_ex.ex index 8665f63f..285f4964 100644 --- a/lib/prom_ex.ex +++ b/lib/prom_ex.ex @@ -190,6 +190,7 @@ defmodule PromEx do manual_metrics_start_delay: manual_metrics_start_delay, drop_metrics_groups: drop_metrics_groups, ets_flush_interval: ets_flush_interval, + ets_flush_timeout: ets_flush_timeout, grafana_config: grafana_config, grafana_agent_config: grafana_agent_config, metrics_server_config: metrics_server_config @@ -216,7 +217,7 @@ defmodule PromEx do # Start the relevant child processes depending on configuration children = [] - |> PromEx.ets_cron_flusher_child_spec(__MODULE__, ets_flush_interval, unquote(ets_cron_flusher_name)) + |> PromEx.ets_cron_flusher_child_spec(__MODULE__, ets_flush_interval, ets_flush_timeout, unquote(ets_cron_flusher_name)) |> PromEx.metrics_collector_child_spec(unquote(store), telemetry_metrics, unquote(metrics_collector_name)) |> PromEx.manual_metrics_child_spec( manual_metrics, @@ -400,10 +401,13 @@ defmodule PromEx do end @doc false - def ets_cron_flusher_child_spec(acc, prom_ex_module, ets_flush_interval, process_name) do + def ets_cron_flusher_child_spec(acc, prom_ex_module, ets_flush_interval, ets_flush_timeout, process_name) do spec = {PromEx.ETSCronFlusher, - name: process_name, prom_ex_module: prom_ex_module, ets_flush_interval: ets_flush_interval} + name: process_name, + prom_ex_module: prom_ex_module, + ets_flush_interval: ets_flush_interval, + ets_flush_timeout: ets_flush_timeout} [spec | acc] end diff --git a/lib/prom_ex/config.ex b/lib/prom_ex/config.ex index dc1f23f2..fff5df71 100644 --- a/lib/prom_ex/config.ex +++ b/lib/prom_ex/config.ex @@ -91,6 +91,11 @@ defmodule PromEx.Config do that periodically compacts ETS. This config value determines how often ETS should be compacted. Default value: `7_500` + * `ets_flush_timeout` - The maximum time in milliseconds to wait for an ETS flush to complete. + If the flush takes longer than this value, the task is shut down and a warning is logged. + On busy systems or shared-cpu machines, `get_metrics/1` can be slow under load, so you may + need to increase this value. Default value: `10_000` + * `:grafana` - This key contains the configuration information for connecting to Grafana. Its configuration options are: @@ -248,6 +253,7 @@ defmodule PromEx.Config do manual_metrics_start_delay: :no_delay | pos_integer(), drop_metrics_groups: MapSet.t(), ets_flush_interval: :integer, + ets_flush_timeout: :integer, grafana_config: map(), grafana_agent_config: map(), metrics_server_config: map() @@ -258,6 +264,7 @@ defmodule PromEx.Config do :manual_metrics_start_delay, :drop_metrics_groups, :ets_flush_interval, + :ets_flush_timeout, :grafana_config, :grafana_agent_config, :metrics_server_config @@ -289,6 +296,7 @@ defmodule PromEx.Config do manual_metrics_start_delay: Keyword.get(opts, :manual_metrics_start_delay, :no_delay), drop_metrics_groups: opts |> Keyword.get(:drop_metrics_groups, []) |> MapSet.new(), ets_flush_interval: Keyword.get(opts, :ets_flush_interval, 7_500), + ets_flush_timeout: Keyword.get(opts, :ets_flush_timeout, 10_000), grafana_config: grafana_config, grafana_agent_config: grafana_agent_config, metrics_server_config: metrics_server_config diff --git a/lib/prom_ex/ets_cron_flusher.ex b/lib/prom_ex/ets_cron_flusher.ex index 64cba7c9..33694230 100644 --- a/lib/prom_ex/ets_cron_flusher.ex +++ b/lib/prom_ex/ets_cron_flusher.ex @@ -2,16 +2,13 @@ defmodule PromEx.ETSCronFlusher do @moduledoc """ This module is used to regularly flush ETS of any buffered distribution type metrics (see https://github.com/beam-telemetry/telemetry_metrics_prometheus_core/blob/main/lib/core.ex#L25-L28) - for more information. At the moment the flush interval is not configurable - but that could change in the future. + for more information. """ use GenServer require Logger - @flush_timeout 10_000 - @doc """ Used to start the `PromEx.ETSCronFlusher` process. """ @@ -51,7 +48,7 @@ defmodule PromEx.ETSCronFlusher do end @impl true - def handle_info(:flush_ets, state) do + def handle_info(:flush_ets, %{ets_flush_timeout: timeout} = state) do # In order to avoid leaking large binaries of metrics, the flush should take place # inside of an ephemeral task so that the heap memory is reclaimed when the process # dies @@ -60,12 +57,12 @@ defmodule PromEx.ETSCronFlusher do PromEx.get_metrics(state.prom_ex_module) end) - case Task.yield(flush_task, @flush_timeout) || Task.shutdown(flush_task) do + case Task.yield(flush_task, timeout) || Task.shutdown(flush_task) do {:ok, _result} -> :ok nil -> - Logger.warning("[PromEx.ETSCronFlusher] ETS flush timed out after #{@flush_timeout}ms") + Logger.warning("[PromEx.ETSCronFlusher] ETS flush timed out after #{timeout}ms") end timer_ref = schedule_flush(state) diff --git a/test/prom_ex/ets_cron_flusher_test.exs b/test/prom_ex/ets_cron_flusher_test.exs index 1a625404..7c520fd8 100644 --- a/test/prom_ex/ets_cron_flusher_test.exs +++ b/test/prom_ex/ets_cron_flusher_test.exs @@ -27,6 +27,7 @@ defmodule PromEx.ETSCronFlusherTest do disabled: false, drop_metrics_groups: MapSet.new(), ets_flush_interval: 2_500, + ets_flush_timeout: 10_000, grafana_agent_config: :disabled, grafana_config: :disabled, manual_metrics_start_delay: :no_delay, From a50095c22e531aa4b6600324f7e52d4debde8062 Mon Sep 17 00:00:00 2001 From: Brieuc Tribouillet Date: Fri, 20 Mar 2026 10:48:07 +0800 Subject: [PATCH 3/3] fix(dashboard_uploader): raise invalid JSON and upload failures to Logger.error for Sentry capture --- lib/prom_ex/dashboard_uploader.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/prom_ex/dashboard_uploader.ex b/lib/prom_ex/dashboard_uploader.ex index 7837699d..de279b1d 100644 --- a/lib/prom_ex/dashboard_uploader.ex +++ b/lib/prom_ex/dashboard_uploader.ex @@ -66,8 +66,8 @@ defmodule PromEx.DashboardUploader do upload_dashboard(dashboard_definition, grafana_conn, upload_opts, full_path) %DashboardRenderer{full_path: path, error: error} -> - Logger.info( - "The dashboard definition for #{inspect(path)} is invalid due to the following error: #{inspect(error)}" + Logger.error( + "PromEx.DashboardUploader failed to render dashboard #{inspect(path)}: #{inspect(error)}" ) end end) @@ -133,7 +133,7 @@ defmodule PromEx.DashboardUploader do Logger.info("PromEx.DashboardUploader successfully uploaded #{full_dashboard_path} to Grafana.") {:error, reason} -> - Logger.warning( + Logger.error( "PromEx.DashboardUploader failed to upload #{full_dashboard_path} to Grafana: #{inspect(reason)}" ) end