Fix ETSCronFlusher crash on flush timeout

lib/prom_ex.ex

@@ -190,6 +190,7 @@ defmodule PromEx do
           manual_metrics_start_delay: manual_metrics_start_delay,
           drop_metrics_groups: drop_metrics_groups,
           ets_flush_interval: ets_flush_interval,
+          ets_flush_timeout: ets_flush_timeout,
           grafana_config: grafana_config,
           grafana_agent_config: grafana_agent_config,
           metrics_server_config: metrics_server_config
@@ -216,7 +217,7 @@ defmodule PromEx do
           # Start the relevant child processes depending on configuration
           children =
             []
-            |> PromEx.ets_cron_flusher_child_spec(__MODULE__, ets_flush_interval, unquote(ets_cron_flusher_name))
+            |> PromEx.ets_cron_flusher_child_spec(__MODULE__, ets_flush_interval, ets_flush_timeout, unquote(ets_cron_flusher_name))
             |> PromEx.metrics_collector_child_spec(unquote(store), telemetry_metrics, unquote(metrics_collector_name))
             |> PromEx.manual_metrics_child_spec(
               manual_metrics,
@@ -400,10 +401,13 @@ defmodule PromEx do
   end
 
   @doc false
-  def ets_cron_flusher_child_spec(acc, prom_ex_module, ets_flush_interval, process_name) do
+  def ets_cron_flusher_child_spec(acc, prom_ex_module, ets_flush_interval, ets_flush_timeout, process_name) do
     spec =
       {PromEx.ETSCronFlusher,
-       name: process_name, prom_ex_module: prom_ex_module, ets_flush_interval: ets_flush_interval}
+       name: process_name,
+       prom_ex_module: prom_ex_module,
+       ets_flush_interval: ets_flush_interval,
+       ets_flush_timeout: ets_flush_timeout}
 
     [spec | acc]
   end

lib/prom_ex/config.ex

@@ -91,6 +91,11 @@ defmodule PromEx.Config do
     that periodically compacts ETS. This config value determines how often ETS should be compacted.
     Default value: `7_500`
 
+  * `ets_flush_timeout` - The maximum time in milliseconds to wait for an ETS flush to complete.
+    If the flush takes longer than this value, the task is shut down and a warning is logged.
+    On busy systems or shared-cpu machines, `get_metrics/1` can be slow under load, so you may
+    need to increase this value. Default value: `10_000`
+
   * `:grafana` - This key contains the configuration information for connecting to Grafana. Its
     configuration options are:
 
@@ -248,6 +253,7 @@ defmodule PromEx.Config do
           manual_metrics_start_delay: :no_delay | pos_integer(),
           drop_metrics_groups: MapSet.t(),
           ets_flush_interval: :integer,
+          ets_flush_timeout: :integer,
           grafana_config: map(),
           grafana_agent_config: map(),
           metrics_server_config: map()
@@ -258,6 +264,7 @@ defmodule PromEx.Config do
     :manual_metrics_start_delay,
     :drop_metrics_groups,
     :ets_flush_interval,
+    :ets_flush_timeout,
     :grafana_config,
     :grafana_agent_config,
     :metrics_server_config
@@ -289,6 +296,7 @@ defmodule PromEx.Config do
       manual_metrics_start_delay: Keyword.get(opts, :manual_metrics_start_delay, :no_delay),
       drop_metrics_groups: opts |> Keyword.get(:drop_metrics_groups, []) |> MapSet.new(),
       ets_flush_interval: Keyword.get(opts, :ets_flush_interval, 7_500),
+      ets_flush_timeout: Keyword.get(opts, :ets_flush_timeout, 10_000),
       grafana_config: grafana_config,
       grafana_agent_config: grafana_agent_config,
       metrics_server_config: metrics_server_config

lib/prom_ex/dashboard_uploader.ex

@@ -66,8 +66,8 @@ defmodule PromEx.DashboardUploader do
           upload_dashboard(dashboard_definition, grafana_conn, upload_opts, full_path)
 
         %DashboardRenderer{full_path: path, error: error} ->
-          Logger.info(
-            "The dashboard definition for #{inspect(path)} is invalid due to the following error: #{inspect(error)}"
+          Logger.error(
+            "PromEx.DashboardUploader failed to render dashboard #{inspect(path)}: #{inspect(error)}"
           )
       end
     end)
@@ -133,7 +133,7 @@ defmodule PromEx.DashboardUploader do
         Logger.info("PromEx.DashboardUploader successfully uploaded #{full_dashboard_path} to Grafana.")
 
       {:error, reason} ->
-        Logger.warning(
+        Logger.error(
           "PromEx.DashboardUploader failed to upload #{full_dashboard_path} to Grafana: #{inspect(reason)}"
         )
     end

lib/prom_ex/ets_cron_flusher.ex

@@ -2,13 +2,12 @@ defmodule PromEx.ETSCronFlusher do
   @moduledoc """
   This module is used to regularly flush ETS of any buffered distribution
   type metrics (see https://github.com/beam-telemetry/telemetry_metrics_prometheus_core/blob/main/lib/core.ex#L25-L28)
-  for more information. At the moment the flush interval is not configurable
-  but that could change in the future.
+  for more information.
   """
 
   use GenServer
 
-  @flush_timeout 10_000
+  require Logger
 
   @doc """
   Used to start the `PromEx.ETSCronFlusher` process.
@@ -49,7 +48,7 @@ defmodule PromEx.ETSCronFlusher do
   end
 
   @impl true
-  def handle_info(:flush_ets, state) do
+  def handle_info(:flush_ets, %{ets_flush_timeout: timeout} = state) do
     # In order to avoid leaking large binaries of metrics, the flush should take place
     # inside of an ephemeral task so that the heap memory is reclaimed when the process
     # dies
@@ -58,7 +57,13 @@ defmodule PromEx.ETSCronFlusher do
         PromEx.get_metrics(state.prom_ex_module)
       end)
 
-    Task.await(flush_task, @flush_timeout)
+    case Task.yield(flush_task, timeout) || Task.shutdown(flush_task) do
+      {:ok, _result} ->
+        :ok
+
+      nil ->
+        Logger.warning("[PromEx.ETSCronFlusher] ETS flush timed out after #{timeout}ms")
+    end
 
     timer_ref = schedule_flush(state)
     {:noreply, %{state | timer_ref: timer_ref}}

test/prom_ex/ets_cron_flusher_test.exs

@@ -27,6 +27,7 @@ defmodule PromEx.ETSCronFlusherTest do
         disabled: false,
         drop_metrics_groups: MapSet.new(),
         ets_flush_interval: 2_500,
+        ets_flush_timeout: 10_000,
         grafana_agent_config: :disabled,
         grafana_config: :disabled,
         manual_metrics_start_delay: :no_delay,