diff --git a/lib/prom_ex.ex b/lib/prom_ex.ex index 8665f63f..285f4964 100644 --- a/lib/prom_ex.ex +++ b/lib/prom_ex.ex @@ -190,6 +190,7 @@ defmodule PromEx do manual_metrics_start_delay: manual_metrics_start_delay, drop_metrics_groups: drop_metrics_groups, ets_flush_interval: ets_flush_interval, + ets_flush_timeout: ets_flush_timeout, grafana_config: grafana_config, grafana_agent_config: grafana_agent_config, metrics_server_config: metrics_server_config @@ -216,7 +217,7 @@ defmodule PromEx do # Start the relevant child processes depending on configuration children = [] - |> PromEx.ets_cron_flusher_child_spec(__MODULE__, ets_flush_interval, unquote(ets_cron_flusher_name)) + |> PromEx.ets_cron_flusher_child_spec(__MODULE__, ets_flush_interval, ets_flush_timeout, unquote(ets_cron_flusher_name)) |> PromEx.metrics_collector_child_spec(unquote(store), telemetry_metrics, unquote(metrics_collector_name)) |> PromEx.manual_metrics_child_spec( manual_metrics, @@ -400,10 +401,13 @@ defmodule PromEx do end @doc false - def ets_cron_flusher_child_spec(acc, prom_ex_module, ets_flush_interval, process_name) do + def ets_cron_flusher_child_spec(acc, prom_ex_module, ets_flush_interval, ets_flush_timeout, process_name) do spec = {PromEx.ETSCronFlusher, - name: process_name, prom_ex_module: prom_ex_module, ets_flush_interval: ets_flush_interval} + name: process_name, + prom_ex_module: prom_ex_module, + ets_flush_interval: ets_flush_interval, + ets_flush_timeout: ets_flush_timeout} [spec | acc] end diff --git a/lib/prom_ex/config.ex b/lib/prom_ex/config.ex index dc1f23f2..fff5df71 100644 --- a/lib/prom_ex/config.ex +++ b/lib/prom_ex/config.ex @@ -91,6 +91,11 @@ defmodule PromEx.Config do that periodically compacts ETS. This config value determines how often ETS should be compacted. Default value: `7_500` + * `ets_flush_timeout` - The maximum time in milliseconds to wait for an ETS flush to complete. + If the flush takes longer than this value, the task is shut down and a warning is logged. + On busy systems or shared-cpu machines, `get_metrics/1` can be slow under load, so you may + need to increase this value. Default value: `10_000` + * `:grafana` - This key contains the configuration information for connecting to Grafana. Its configuration options are: @@ -248,6 +253,7 @@ defmodule PromEx.Config do manual_metrics_start_delay: :no_delay | pos_integer(), drop_metrics_groups: MapSet.t(), ets_flush_interval: :integer, + ets_flush_timeout: :integer, grafana_config: map(), grafana_agent_config: map(), metrics_server_config: map() @@ -258,6 +264,7 @@ defmodule PromEx.Config do :manual_metrics_start_delay, :drop_metrics_groups, :ets_flush_interval, + :ets_flush_timeout, :grafana_config, :grafana_agent_config, :metrics_server_config @@ -289,6 +296,7 @@ defmodule PromEx.Config do manual_metrics_start_delay: Keyword.get(opts, :manual_metrics_start_delay, :no_delay), drop_metrics_groups: opts |> Keyword.get(:drop_metrics_groups, []) |> MapSet.new(), ets_flush_interval: Keyword.get(opts, :ets_flush_interval, 7_500), + ets_flush_timeout: Keyword.get(opts, :ets_flush_timeout, 10_000), grafana_config: grafana_config, grafana_agent_config: grafana_agent_config, metrics_server_config: metrics_server_config diff --git a/lib/prom_ex/dashboard_uploader.ex b/lib/prom_ex/dashboard_uploader.ex index 7837699d..de279b1d 100644 --- a/lib/prom_ex/dashboard_uploader.ex +++ b/lib/prom_ex/dashboard_uploader.ex @@ -66,8 +66,8 @@ defmodule PromEx.DashboardUploader do upload_dashboard(dashboard_definition, grafana_conn, upload_opts, full_path) %DashboardRenderer{full_path: path, error: error} -> - Logger.info( - "The dashboard definition for #{inspect(path)} is invalid due to the following error: #{inspect(error)}" + Logger.error( + "PromEx.DashboardUploader failed to render dashboard #{inspect(path)}: #{inspect(error)}" ) end end) @@ -133,7 +133,7 @@ defmodule PromEx.DashboardUploader do Logger.info("PromEx.DashboardUploader successfully uploaded #{full_dashboard_path} to Grafana.") {:error, reason} -> - Logger.warning( + Logger.error( "PromEx.DashboardUploader failed to upload #{full_dashboard_path} to Grafana: #{inspect(reason)}" ) end diff --git a/lib/prom_ex/ets_cron_flusher.ex b/lib/prom_ex/ets_cron_flusher.ex index bd8e91da..33694230 100644 --- a/lib/prom_ex/ets_cron_flusher.ex +++ b/lib/prom_ex/ets_cron_flusher.ex @@ -2,13 +2,12 @@ defmodule PromEx.ETSCronFlusher do @moduledoc """ This module is used to regularly flush ETS of any buffered distribution type metrics (see https://github.com/beam-telemetry/telemetry_metrics_prometheus_core/blob/main/lib/core.ex#L25-L28) - for more information. At the moment the flush interval is not configurable - but that could change in the future. + for more information. """ use GenServer - @flush_timeout 10_000 + require Logger @doc """ Used to start the `PromEx.ETSCronFlusher` process. @@ -49,7 +48,7 @@ defmodule PromEx.ETSCronFlusher do end @impl true - def handle_info(:flush_ets, state) do + def handle_info(:flush_ets, %{ets_flush_timeout: timeout} = state) do # In order to avoid leaking large binaries of metrics, the flush should take place # inside of an ephemeral task so that the heap memory is reclaimed when the process # dies @@ -58,7 +57,13 @@ defmodule PromEx.ETSCronFlusher do PromEx.get_metrics(state.prom_ex_module) end) - Task.await(flush_task, @flush_timeout) + case Task.yield(flush_task, timeout) || Task.shutdown(flush_task) do + {:ok, _result} -> + :ok + + nil -> + Logger.warning("[PromEx.ETSCronFlusher] ETS flush timed out after #{timeout}ms") + end timer_ref = schedule_flush(state) {:noreply, %{state | timer_ref: timer_ref}} diff --git a/test/prom_ex/ets_cron_flusher_test.exs b/test/prom_ex/ets_cron_flusher_test.exs index 1a625404..7c520fd8 100644 --- a/test/prom_ex/ets_cron_flusher_test.exs +++ b/test/prom_ex/ets_cron_flusher_test.exs @@ -27,6 +27,7 @@ defmodule PromEx.ETSCronFlusherTest do disabled: false, drop_metrics_groups: MapSet.new(), ets_flush_interval: 2_500, + ets_flush_timeout: 10_000, grafana_agent_config: :disabled, grafana_config: :disabled, manual_metrics_start_delay: :no_delay,