SapMachine #2392: Add load average to vitals

src/hotspot/os/aix/vitals_aix.cpp

@@ -30,11 +30,17 @@
 
 namespace sapmachine_vitals {
 
+static Column* g_col_system_load_average = nullptr;
+
 bool platform_columns_initialize() {
+  g_col_system_load_average =
+      define_column<PlainValueColumn>("system", nullptr, "la", "Load average in the sample interval in percent", true);
+
   return true;
 }
 
-void sample_platform_values(Sample* record) {
+void sample_platform_values(Sample* sample, Sample* long_term_sample) {
+  set_load_average(g_col_system_load_average, get_load_avg_from_os_interface(), sample, long_term_sample);
 }
 
 } // namespace sapmachine_vitals

src/hotspot/os/bsd/vitals_bsd.cpp

@@ -30,11 +30,17 @@
 
 namespace sapmachine_vitals {
 
+static Column* g_col_system_load_average = nullptr;
+
 bool platform_columns_initialize() {
+  g_col_system_load_average =
+      define_column<PlainValueColumn>("system", nullptr, "la", "Load average in the sample interval in percent", true);
+
   return true;
 }
 
-void sample_platform_values(Sample* record) {
+void sample_platform_values(Sample* sample, Sample* long_term_sample) {
+  set_load_average(g_col_system_load_average, get_load_avg_from_os_interface(), sample, long_term_sample);
 }
 
 } // namespace sapmachine_vitals

src/hotspot/os/linux/vitals_linux.cpp

@@ -105,6 +105,7 @@ static Column* g_col_system_num_threads = nullptr;
 
 static Column* g_col_system_num_procs_running = nullptr;
 static Column* g_col_system_num_procs_blocked = nullptr;
+static Column* g_col_system_load_average = nullptr;
 
 static bool g_show_cgroup_info = false;
 static Column* g_col_system_cgrp_limit_in_bytes = nullptr;
@@ -179,6 +180,8 @@ bool platform_columns_initialize() {
       define_column<PlainValueColumn>(system_cat, nullptr, "tr", "Number of threads running", true);
   g_col_system_num_procs_blocked =
       define_column<PlainValueColumn>(system_cat, nullptr, "tb", "Number of threads blocked on disk IO", true);
+  g_col_system_load_average =
+      define_column<PlainValueColumn>(system_cat, nullptr, "la", "Load average in the sample interval in percent", true);
 
   g_col_system_cpu_user =
       define_column<CPUTimeColumn>(system_cat, "cpu", "us", "CPU user time [host]", true);
@@ -261,7 +264,7 @@ static void set_value_in_sample(Column* col, Sample* sample, value_t val) {
   }
 }
 
-void sample_platform_values(Sample* sample) {
+void sample_platform_values(Sample* sample, Sample* long_term_sample) {
 
   int idx = 0;
 
@@ -286,6 +289,7 @@ void sample_platform_values(Sample* sample) {
 
   set_value_in_sample(g_col_system_num_procs_running, sample, OSWrapper::syst_tr());
   set_value_in_sample(g_col_system_num_procs_blocked, sample, OSWrapper::syst_tb());
+  set_load_average(g_col_system_load_average, OSWrapper::syst_load_average(), sample, long_term_sample);
 
   // cgroups business
   if (g_show_cgroup_info) {

src/hotspot/os/linux/vitals_linux_oswrapper.cpp

@@ -27,6 +27,7 @@
 #include "logging/log.hpp"
 #include "osContainer_linux.hpp"
 #include "runtime/os.hpp"
+#include "runtime/timerTrace.hpp"
 #include "utilities/globalDefinitions.hpp"
 #include "vitals/vitals_internals.hpp"
 #include "vitals_linux_oswrapper.hpp"
@@ -58,6 +59,7 @@ static const int num_seconds_until_update = 1;
 
 class ProcFile {
   char* _buf;
+  const char* _filename;
 
   // To keep the code simple, I just use a fixed sized buffer.
   enum { bufsize = 64*K };
@@ -66,6 +68,7 @@ class ProcFile {
 
   ProcFile() : _buf(nullptr) {
     _buf = (char*)os::malloc(bufsize, mtInternal);
+    _filename = "<invalid>";
   }
 
   ~ProcFile () {
@@ -84,20 +87,22 @@ class ProcFile {
     _buf[bytes_read] = '\0';
 
     ::fclose(f);
+    // All filenames we get are live for the time we need it. No need to strdup.
+    _filename = filename;
 
     return bytes_read > 0 && bytes_read < bufsize;
   }
 
   const char* text() const { return _buf; }
 
   // Utility function; parse a number string as value_t
-  static value_t as_value(const char* text, size_t scale = 1) {
+  static value_t as_value(const char* prefix, const char* text, size_t scale = 1) {
     value_t value;
     errno = 0;
     char* endptr = nullptr;
     value = (value_t)::strtoll(text, &endptr, 10);
     if (endptr == text || errno != 0) {
-      log_debug(vitals, os)("Failed to parse \"%s\"", text);
+      log_debug(vitals, os)("Failed to parse %s: \"%s\"", prefix, text);
       value = INVALID_VALUE;
     } else {
       value *= scale;
@@ -108,7 +113,7 @@ class ProcFile {
   // Return the start of the file, as number. Useful for proc files which
   // contain a single number. Returns INVALID_VALUE if value did not parse
   value_t as_value(size_t scale = 1) const {
-    return as_value(_buf, scale);
+    return as_value(_filename, _buf, scale);
   }
 
   const char* get_prefixed_line(const char* prefix) const {
@@ -121,7 +126,7 @@ class ProcFile {
     if (s != nullptr) {
       errno = 0;
       const char* p = s + ::strlen(prefix);
-      value = as_value(p, scale);
+      value = as_value(prefix, p, scale);
       log_trace(vitals, os)("Reading \"%s\": %llu", prefix, (unsigned long long) value);
     } else {
       log_debug(vitals, os)("Could not find prefix \"%s\"", prefix);
@@ -491,6 +496,7 @@ ALL_VALUES_DO(RESETVAL)
   // Number of processes: iterate over /proc/<pid> and count.
   // Number of threads: read "num_threads" from /proc/<pid>/stat
   {
+    TraceTime timer("Iterating all processes", TRACETIME_LOG(Debug, vitals, os));
     DIR* d = ::opendir("/proc");
     if (d != nullptr) {
       value_t v_p = 0;
@@ -556,6 +562,14 @@ ALL_VALUES_DO(RESETVAL)
   }
 #endif // __GLIBC__
 
+  if ((VitalsSampleInterval < 30) && (_syst_tr != INVALID_VALUE)) {
+    // For short sample times we use the number of runnable and running threads
+    // to approximate the load average in that interval.
+    _syst_load_average = (value_t) MAX2(0.0, _syst_tr * get_proc_scale_factor());
+  } else {
+    _syst_load_average = get_load_avg_from_os_interface();
+  }
+
   first_call = false;
 
 }

src/hotspot/os/linux/vitals_linux_oswrapper.hpp

@@ -48,6 +48,7 @@ class OSWrapper {
 	  f(syst_t) \
 	  f(syst_tr) \
 	  f(syst_tb) \
+	  f(syst_load_average) \
 	  f(syst_cpu_us) \
 	  f(syst_cpu_sy) \
 	  f(syst_cpu_id) \

src/hotspot/os/windows/vitals_windows.cpp

@@ -23,27 +23,140 @@
  *
  */
 
+#include "logging/log.hpp"
 #include "runtime/os.hpp"
+#include "runtime/timerTrace.hpp"
 #include "utilities/debug.hpp"
 #include "utilities/globalDefinitions.hpp"
+#include "utilities/ostream.hpp"
 #include "vitals/vitals_internals.hpp"
+#include "pdh_interface.hpp"
 
 #include <psapi.h>
 
 namespace sapmachine_vitals {
 
+static const DWORD PDH_SYSTEM_IDX = 2;
+static const DWORD PDH_PROCESSOR_TIME_IDX = 6;
+static const DWORD PDH_PROCESSOR_QUEUE_LENGTH_IDX = 44;
+static const DWORD PDH_PROCESSOR_IDX = 238;
+
 static Column* g_col_system_memoryload = nullptr;
 static Column* g_col_system_avail_phys = nullptr;
+static Column* g_col_system_load_average = nullptr;
 static Column* g_col_process_working_set_size = nullptr;
 static Column* g_col_process_commit_charge = nullptr;
 
+static bool log_pdh(const char* operation, PDH_STATUS status) {
+  if (status != ERROR_SUCCESS) {
+    log_debug(vitals)("pdh operation '%s' failed with error code %x", operation, status);
+    return false;
+  }
+
+  return true;
+}
+
+static bool has_loadavg = false;
+static double proc_scale_factor = 1.0;
+
+static HQUERY query;
+static HCOUNTER queue_length_counter, processor_time_counter;
+static PDH_FMT_COUNTERVALUE queue_length, processor_time;
+
+static bool add_pdh_string_from_index(DWORD index, stringStream* ss) {
+  DWORD size = 0;
+
+  if (PdhDll::PdhLookupPerfNameByIndex(nullptr, index, nullptr, &size) != PDH_MORE_DATA) {
+    return false;
+  }
+
+  char* pdh_string = NEW_C_HEAP_ARRAY(char, size, mtInternal);
+  pdh_string[size - 1] = '\0';
+  PDH_STATUS status = PdhDll::PdhLookupPerfNameByIndex(nullptr, index, pdh_string, &size);
+
+  if (status == ERROR_SUCCESS) {
+    ss->print_raw(pdh_string);
+  } else {
+    log_pdh("Converting index failed", status);
+  }
+
+  FREE_C_HEAP_ARRAY(pdh_string);
+
+  return status == ERROR_SUCCESS;
+}
+
+static double get_load_average_impl(bool first_call) {
+  double load_avg = -1;
+
+  if (first_call) {
+    stringStream queue_lengt_counter_name;
+    queue_lengt_counter_name.put('\\');
+    bool success = add_pdh_string_from_index(PDH_SYSTEM_IDX, &queue_lengt_counter_name);
+    queue_lengt_counter_name.put('\\');
+    success = success && add_pdh_string_from_index(PDH_PROCESSOR_QUEUE_LENGTH_IDX, &queue_lengt_counter_name);
+
+    stringStream processor_time_counter_name;
+    processor_time_counter_name.put('\\');
+    success = success && add_pdh_string_from_index(PDH_PROCESSOR_IDX, &processor_time_counter_name);
+    processor_time_counter_name.print_raw("(_Total)\\");
+    success = success && add_pdh_string_from_index(PDH_PROCESSOR_TIME_IDX, &processor_time_counter_name);
+
+    if (!success) {
+      log_debug(vitals)("Could not create the localized counters: '%s', '%s'", queue_lengt_counter_name.base(), processor_time_counter_name.base());
+      return load_avg;
+    }
+
+    if (log_pdh("open query", PdhDll::PdhOpenQuery(nullptr, 0, &query))) {
+      has_loadavg = log_pdh("add queue length", PdhDll::PdhAddCounter(query, queue_lengt_counter_name.base(), 0, &queue_length_counter)) &&
+        log_pdh("add processor time", PdhDll::PdhAddCounter(query, processor_time_counter_name.base(), 0, &processor_time_counter)) &&
+        log_pdh("collect data", PdhDll::PdhCollectQueryData(query));
+
+      if (!has_loadavg) {
+        log_pdh("close query", PdhDll::PdhCloseQuery(query));
+      }
+    }
+
+    proc_scale_factor = 100.0 / MAX2(1, os::processor_count());
+  }
+  else {
+    TraceTime timer("Getting the counter values", TRACETIME_LOG(Debug, vitals, os));
+
+    if (log_pdh("collect data", PdhDll::PdhCollectQueryData(query)) &&
+      log_pdh("format queue length", PdhDll::PdhGetFormattedCounterValue(queue_length_counter, PDH_FMT_DOUBLE, nullptr, &queue_length)) &&
+      log_pdh("format processor time", PdhDll::PdhGetFormattedCounterValue(processor_time_counter, PDH_FMT_DOUBLE, nullptr, &processor_time))) {
+      log_debug(vitals)("Queue length %d, processor time %d", (int)queue_length.doubleValue, (int)processor_time.doubleValue);
+      load_avg = processor_time.doubleValue + queue_length.doubleValue * proc_scale_factor;
+    }
+  }
+
+  return load_avg;
+}
+
+static void initialize_pdh() {
+  if (!PdhDll::PdhAttach()) {
+    log_debug(vitals)("Could not attach pdh lib.");
+  } else {
+    get_load_average_impl(true);
+  }
+
+  if (!has_loadavg) {
+    PdhDll::PdhDetach();
+  }
+}
+
 bool platform_columns_initialize() {
+  initialize_pdh();
+
   g_col_system_memoryload =
       define_column<PlainValueColumn>("system", nullptr, "mload", "Approximate percentage of physical memory that is in use.", true, MAX);
 
   // MEMORYSTATUSEX ullAvailPhys
   g_col_system_avail_phys =
       define_column<MemorySizeColumn>("system", nullptr, "avail-phys", "Amount of physical memory currently available.", true, MIN);
+
+  g_col_system_load_average =
+    define_column<PlainValueColumn>("system", nullptr, "la", "Load average in the sample interval in percent.", has_loadavg, MAX);
+
   // PROCESS_MEMORY_COUNTERS_EX WorkingSetSize
   g_col_process_working_set_size =
       define_column<MemorySizeColumn>("system", nullptr, "wset", "Working set size", true);
@@ -62,7 +175,15 @@ static void set_value_in_sample(Column* col, Sample* sample, value_t val) {
   }
 }
 
-void sample_platform_values(Sample* sample) {
+static value_t get_load_average() {
+  if (!has_loadavg) {
+    return INVALID_VALUE;
+  }
+
+  return get_load_average_impl(false);
+}
+
+void sample_platform_values(Sample* sample, Sample* long_term_sample) {
   MEMORYSTATUSEX mse;
   mse.dwLength = sizeof(mse);
   if (::GlobalMemoryStatusEx(&mse)) {
@@ -76,6 +197,8 @@ void sample_platform_values(Sample* sample) {
     set_value_in_sample(g_col_process_working_set_size, sample, cnt.WorkingSetSize);
     set_value_in_sample(g_col_process_commit_charge, sample, cnt.PagefileUsage);
   }
+
+  set_load_average(g_col_system_load_average, get_load_average(), sample, long_term_sample);
 }
 
 } // namespace sapmachine_vitals

src/hotspot/share/runtime/threads.cpp

@@ -859,6 +859,9 @@ jint Threads::create_vm(JavaVMInitArgs* args, bool* canTryAgain) {
     sapmachine_vitals::initialize_himem_report_facility();
   }
 #endif // LINUX
+  if (EnableVitals) {
+    sapmachine_vitals::begin_sampling();
+  }
 
 #if INCLUDE_MANAGEMENT
   bool start_agent = true;