diff --git a/lib/sentry/metrics.ex b/lib/sentry/metrics.ex index 903e7067..d1b804c8 100644 --- a/lib/sentry/metrics.ex +++ b/lib/sentry/metrics.ex @@ -41,6 +41,10 @@ defmodule Sentry.Metrics do `atom`, `atom_used`, `binary`, `code` and `ets`, in bytes * `elixir.runtime.run_queue.*` — `total`, `cpu` and `io`, how many processes are waiting to run + * `elixir.runtime.process.*`, `elixir.runtime.atom.*` and `elixir.runtime.port.*` — + a `count`, the hard VM `limit`, and the `utilization` ratio between them. The + `limit` and `utilization` gauges need telemetry_poller 1.3.0 or later, which is + when it started measuring the limits; on older versions only `count` is reported. ### Collection Frequency diff --git a/lib/sentry/metrics/runtime.ex b/lib/sentry/metrics/runtime.ex index fa1d4ccb..582a5515 100644 --- a/lib/sentry/metrics/runtime.ex +++ b/lib/sentry/metrics/runtime.ex @@ -8,11 +8,18 @@ defmodule Sentry.Metrics.Runtime do @memory_event [:vm, :memory] @run_queue_event [:vm, :total_run_queue_lengths] + @system_counts_event [:vm, :system_counts] - @events [@memory_event, @run_queue_event] + @events [@memory_event, @run_queue_event, @system_counts_event] @run_queue_keys [:total, :cpu, :io] + @system_counts [ + {"process", :process_count, :process_limit}, + {"atom", :atom_count, :atom_limit}, + {"port", :port_count, :port_limit} + ] + @memory_keys [ :total, :processes, @@ -55,6 +62,29 @@ defmodule Sentry.Metrics.Runtime do report_measured(config, measurements, @run_queue_keys, "elixir.runtime.run_queue", nil) end + def handle_event(@system_counts_event, measurements, _metadata, config) do + Enum.each(@system_counts, fn {name, count_key, limit_key} -> + report_count(config, name, measurements[count_key], measurements[limit_key]) + end) + end + + defp report_count(_config, _name, nil, _limit), do: :ok + + defp report_count(config, name, count, limit) do + gauge(config, "elixir.runtime.#{name}.count", count, nil) + report_limit(config, name, count, limit) + end + + defp report_limit(_config, _name, _count, nil), do: :ok + + defp report_limit(config, name, count, limit) do + gauge(config, "elixir.runtime.#{name}.limit", limit, nil) + gauge(config, "elixir.runtime.#{name}.utilization", ratio(count, limit), "ratio") + end + + defp ratio(_count, 0), do: 0.0 + defp ratio(count, limit), do: count / limit + defp report_measured(config, measurements, keys, prefix, unit) do Enum.each(Map.take(measurements, keys), fn {key, value} -> gauge(config, "#{prefix}.#{key}", value, unit) diff --git a/test/sentry/metrics/runtime_test.exs b/test/sentry/metrics/runtime_test.exs index a09b80a6..03e0d57d 100644 --- a/test/sentry/metrics/runtime_test.exs +++ b/test/sentry/metrics/runtime_test.exs @@ -55,6 +55,36 @@ defmodule Sentry.Metrics.RuntimeTest do end end + describe "system count metrics" do + @system_counts %{ + process_count: 100, + process_limit: 1_000, + atom_count: 50, + atom_limit: 500, + port_count: 4, + port_limit: 200 + } + + test "reports the count, the limit and the ratio between them" do + metrics = emit([:vm, :system_counts], @system_counts) + + for {name, count, limit} <- [{"process", 100, 1_000}, {"atom", 50, 500}, {"port", 4, 200}] do + assert find_metric!(metrics, "elixir.runtime.#{name}.count").value == count + assert find_metric!(metrics, "elixir.runtime.#{name}.limit").value == limit + + utilization = find_metric!(metrics, "elixir.runtime.#{name}.utilization") + assert utilization.value == count / limit + assert utilization.unit == "ratio" + end + end + + test "omits the limit and the ratio when the poller does not report limits" do + metrics = emit([:vm, :system_counts], %{process_count: 100}) + + assert [%{name: "elixir.runtime.process.count", value: 100}] = metrics + end + end + describe "metric attributes" do test "tags every metric with the runtime metrics origin" do for metric <- emit_memory() do @@ -88,7 +118,7 @@ defmodule Sentry.Metrics.RuntimeTest do describe "wiring against a real telemetry_poller" do test "maps the builtin measurements onto Sentry gauges" do attach() - poller = start_idle_poller([:memory, :total_run_queue_lengths]) + poller = start_idle_poller([:memory, :total_run_queue_lengths, :system_counts]) :ok = SentryTest.allow_sentry_reports(self(), poller) collect_once(poller) @@ -97,6 +127,8 @@ defmodule Sentry.Metrics.RuntimeTest do assert metric.value > 0 assert_sentry_metric(:gauge, name: "elixir.runtime.run_queue.total") + assert_sentry_metric(:gauge, name: "elixir.runtime.process.count") + assert_sentry_metric(:gauge, name: "elixir.runtime.process.limit") end end diff --git a/test_integrations/phoenix_app/test/phoenix_app/runtime_metrics_test.exs b/test_integrations/phoenix_app/test/phoenix_app/runtime_metrics_test.exs index 7a6cfe28..41f03e02 100644 --- a/test_integrations/phoenix_app/test/phoenix_app/runtime_metrics_test.exs +++ b/test_integrations/phoenix_app/test/phoenix_app/runtime_metrics_test.exs @@ -83,6 +83,41 @@ defmodule Sentry.Integrations.Phoenix.RuntimeMetricsTest do end end + describe "system count metrics from a real telemetry_poller" do + setup do + SentryTest.setup_sentry() + :ok + end + + test "reports the real counts against the limits the VM enforces" do + metrics = collect_runtime_metrics([:system_counts]) + + for name <- ["process", "atom", "port"] do + count = find_metric!(metrics, "elixir.runtime.#{name}.count").value + limit = find_metric!(metrics, "elixir.runtime.#{name}.limit").value + utilization = find_metric!(metrics, "elixir.runtime.#{name}.utilization") + + assert count > 0 + assert count < limit + assert utilization.value == count / limit + assert utilization.unit == "ratio" + end + end + + test "forwards the hard limits telemetry_poller measures" do + metrics = collect_runtime_metrics([:system_counts]) + + for {name, key} <- [ + {"process", :process_limit}, + {"atom", :atom_limit}, + {"port", :port_limit} + ] do + assert find_metric!(metrics, "elixir.runtime.#{name}.limit").value == + :erlang.system_info(key) + end + end + end + describe "the application wiring" do test "attaches the runtime metrics handler at boot" do handler_ids = [:vm, :memory] |> :telemetry.list_handlers() |> Enum.map(& &1.id)