Add a configurable "significant error" metric

Change-Id: Idda59272a029ffcbc029f9bb167508d7bd5e6e21
Signed-off-by: Dave Barach <dave@barachs.net>

gmod/gmod/mod_vpp.c

@@ -70,7 +70,7 @@ static g_val_t vpp_metric_handler (int metric_index)
 {
     g_val_t val;
     pid_t *vpp_pidp;
-    f64 *vector_ratep, *vpp_rx_ratep;
+    f64 *vector_ratep, *vpp_rx_ratep, *sig_error_ratep;
 
     switch (metric_index) {
     case 0:
@@ -108,6 +108,16 @@ static g_val_t vpp_metric_handler (int metric_index)
             val.d = 0.0;
         break;
 
+    case 3:
+        sig_error_ratep = svmdb_local_get_vec_variable 
+            (svmdb_client, "vpp_sig_error_rate", sizeof (*vector_ratep));
+        if (sig_error_ratep) {
+            val.d = *sig_error_ratep;
+            vec_free (sig_error_ratep);
+        } else
+            val.d = 0.0;
+        break;
+
     default:
         val.d = 0.0; 
     }
@@ -125,6 +135,10 @@ static Ganglia_25metric vpp_metric_info[] =
     {0, "Input_Rate", 100, GANGLIA_VALUE_DOUBLE, "5 sec RX rate", 
      "both", "%.1f", 
      UDP_HEADER_SIZE+8, "VPP Aggregate RX Rate"},
+    {0, "Sig_Error_Rate", 100, GANGLIA_VALUE_DOUBLE, 
+     "5 sec significant error rate", 
+     "both", "%.1f", 
+     UDP_HEADER_SIZE+8, "VPP Significant Error Rate"},
     {0, NULL}
 };
 

gmod/gmod/vpp.conf

@@ -21,6 +21,11 @@ collection_group {
  metric {
     name = "Input_Rate"
     value_threshold = 10000.0
-    title = "VPP Aggregate RX rate"
+    title = "VPP Aggregate RX Rate"
+  }
+ metric {
+    name = "Sig_Error_Rate"
+    value_threshold = 10.0
+    title = "VPP Significant Error Rate"
   }
 }

svm/svmdb.c

@@ -472,9 +472,9 @@ void svmdb_local_dump_vecs (svmdb_client_t *client)
     hash_foreach_mem(key, value, h, 
     ({
         svmdb_value_t *v = pool_elt_at_index (shm->values, value);
-        (void) fformat(stdout, "%s:\n %U\n", key, 
+        (void) fformat(stdout, "%s:\n %U (%.2f)\n", key, 
                        format_hex_bytes, v->value, 
-                       vec_len(v->value)*v->elsize);
+                       vec_len(v->value)*v->elsize, ((f64 *)(v->value))[0]);
     }));
 
     region_unlock (client->db_rp);

vlib/vlib/error.c

@@ -215,12 +215,18 @@ show_errors (vlib_main_t * vm,
   int verbose = 0;
   u64 * sums = 0;
 
-  if (unformat (input, "verbose"))
+  if (unformat (input, "verbose %d", &verbose))
+    ;
+  else if (unformat (input, "verbose"))
     verbose = 1;
 
   vec_validate(sums, vec_len(em->counters));
 
-  vlib_cli_output (vm, "%=16s%=40s%=20s", "Count", "Node", "Reason");
+  if (verbose)
+    vlib_cli_output (vm, "%=10s%=40s%=20s%=6s", "Count", "Node", "Reason", 
+                     "Index");
+  else
+    vlib_cli_output (vm, "%=10s%=40s%=6s", "Count", "Node", "Reason");
 
   foreach_vlib_main(({
     em = &this_vlib_main->error_main;
@@ -239,10 +245,15 @@ show_errors (vlib_main_t * vm,
 	      c -= em->counters_last_clear[i];
 	    sums[i] += c;
 
-	    if (c == 0 || !verbose)
+	    if (c == 0 && verbose < 2)
 	      continue;
 
-	    vlib_cli_output (vm, "%16Ld%=40v%s", c, n->name, em->error_strings_heap[i]);
+            if (verbose)
+              vlib_cli_output (vm, "%10Ld%=40v%=20s%=6d", c, n->name, 
+                               em->error_strings_heap[i], i);
+            else
+              vlib_cli_output (vm, "%10d%=40v%s", c, n->name, 
+                               em->error_strings_heap[i]);
 	  }
       }
     index++;
@@ -258,7 +269,11 @@ show_errors (vlib_main_t * vm,
 	{
 	  i = n->error_heap_index + code;
 	  if (sums[i])
-	    vlib_cli_output (vm, "%16Ld%=40v%s", sums[i], n->name, em->error_strings_heap[i]);
+            {
+              if (verbose)
+                vlib_cli_output (vm, "%10Ld%=40v%=20s%=10d", sums[i], n->name, 
+                                 em->error_strings_heap[i], i);
+            }
 	}
     }
 

vpp/api/gmon.c

@@ -40,37 +40,69 @@
 
 #include <vlib/vlib.h>
 #include <vlib/unix/unix.h>
+#include <vnet/api_errno.h>
 
 #include <svmdb.h>
 
 typedef struct {
-    svmdb_client_t *svmdb_client;
+  svmdb_client_t *svmdb_client;
-    f64 *vector_rate_ptr;
+  f64 *vector_rate_ptr;
-    f64 *input_rate_ptr;
+  f64 *input_rate_ptr;
-    pid_t *vpef_pid_ptr;
+  f64 *sig_error_rate_ptr;
-    vlib_main_t *vlib_main;
+  pid_t *vpef_pid_ptr;
+  u64 last_sig_errors;
+  u64 current_sig_errors;
+  u64 * sig_error_bitmap;
+  vlib_main_t *vlib_main;
+  vlib_main_t ** my_vlib_mains;
+
 } gmon_main_t;
 
 #if DPDK == 0
 static inline u64 vnet_get_aggregate_rx_packets (void)
 { return 0; }
 #else
+#include <vlib/vlib.h>
 #include <vnet/vnet.h>
 #include <vnet/devices/dpdk/dpdk.h>
 #endif
 
 gmon_main_t gmon_main;
 
+static u64 get_significant_errors(gmon_main_t * gm)
+{
+  vlib_main_t * this_vlib_main;
+  vlib_error_main_t * em;
+  uword code;
+  int vm_index;
+  u64 significant_errors = 0;
+
+  clib_bitmap_foreach (code, gm->sig_error_bitmap,
+  ({
+    for (vm_index = 0; vm_index < vec_len (gm->my_vlib_mains); vm_index++)
+      {
+        this_vlib_main = gm->my_vlib_mains[vm_index];
+        em = &this_vlib_main->error_main;
+        significant_errors += em->counters[code] -
+          ((vec_len(em->counters_last_clear) > code) ?
+           em->counters_last_clear[code] : 0);
+      }
+  }));
+
+  return (significant_errors);
+}
+
 static uword
 gmon_process (vlib_main_t * vm,
               vlib_node_runtime_t * rt,
               vlib_frame_t * f)
 {
     f64 vector_rate;
-    u64 input_packets, last_input_packets;
+    u64 input_packets, last_input_packets, new_sig_errors;
     f64 last_runtime, dt, now;
     gmon_main_t *gm = &gmon_main;
     pid_t vpefpid;
+    int i;
 
     vpefpid = getpid();
     *gm->vpef_pid_ptr = vpefpid;
@@ -81,6 +113,17 @@ gmon_process (vlib_main_t * vm,
     last_runtime = 0.0;
     last_input_packets = 0;
 
+    /* Initial wait for the world to settle down */
+    vlib_process_suspend (vm, 5.0);
+
+    if (vec_len(vlib_mains) == 0)
+      vec_add1(gm->my_vlib_mains, &vlib_global_main);
+    else
+      {
+        for (i = 0; i < vec_len(vlib_mains); i++)
+          vec_add1 (gm->my_vlib_mains, vlib_mains[i]);
+      }
+
     while (1) {
         vlib_process_suspend (vm, 5.0);
         vector_rate = vlib_last_vector_length_per_node (vm);
@@ -91,6 +134,11 @@ gmon_process (vlib_main_t * vm,
         *gm->input_rate_ptr = (f64)(input_packets - last_input_packets) / dt;
         last_runtime = now;
         last_input_packets = input_packets;
+
+        new_sig_errors = get_significant_errors(gm);
+        *gm->sig_error_rate_ptr = 
+            ((f64)(new_sig_errors - gm->last_sig_errors)) / dt;
+        gm->last_sig_errors = new_sig_errors;
     }
 
     return 0; /* not so much */
@@ -124,6 +172,11 @@ gmon_init (vlib_main_t *vm)
                                  "vpp_input_rate", 
                                  (char *)v, sizeof (*v));
     vec_free(v);
+    vec_add1 (v, 0.0);
+    svmdb_local_set_vec_variable(gm->svmdb_client, 
+                                 "vpp_sig_error_rate", 
+                                 (char *)v, sizeof (*v));
+    vec_free(v);
 
     vec_add1 (swp, 0.0);
     svmdb_local_set_vec_variable(gm->svmdb_client, 
@@ -131,7 +184,7 @@ gmon_init (vlib_main_t *vm)
                                  (char *)swp, sizeof (*swp));
     vec_free(swp);
 
-    /* the value cell will never move, so acquire a reference to it */
+    /* the value cells will never move, so acquire references to them */
     gm->vector_rate_ptr = 
         svmdb_local_get_variable_reference (gm->svmdb_client,
                                             SVMDB_NAMESPACE_VEC, 
@@ -140,6 +193,10 @@ gmon_init (vlib_main_t *vm)
         svmdb_local_get_variable_reference (gm->svmdb_client,
                                             SVMDB_NAMESPACE_VEC, 
                                             "vpp_input_rate");
+    gm->sig_error_rate_ptr = 
+        svmdb_local_get_variable_reference (gm->svmdb_client,
+                                            SVMDB_NAMESPACE_VEC, 
+                                            "vpp_sig_error_rate");
     gm->vpef_pid_ptr = 
         svmdb_local_get_variable_reference (gm->svmdb_client,
                                             SVMDB_NAMESPACE_VEC, 
@@ -157,8 +214,64 @@ static clib_error_t *gmon_exit (vlib_main_t *vm)
         *gm->vector_rate_ptr = 0.0;
         *gm->vpef_pid_ptr = 0;
         *gm->input_rate_ptr = 0.0;
+        *gm->sig_error_rate_ptr = 0.0;
         svmdb_unmap (gm->svmdb_client);
     }
     return 0;
 }
 VLIB_MAIN_LOOP_EXIT_FUNCTION (gmon_exit);
+
+static int 
+significant_error_enable_disable (gmon_main_t * gm, 
+                                  u32 index, int enable)
+{
+  vlib_main_t * vm = gm->vlib_main;
+  vlib_error_main_t * em = &vm->error_main;
+
+  if (index >= vec_len (em->counters))
+    return VNET_API_ERROR_NO_SUCH_ENTRY;
+
+  gm->sig_error_bitmap = clib_bitmap_set (gm->sig_error_bitmap, index, enable);
+  return 0;
+}
+
+static clib_error_t *
+set_significant_error_command_fn (vlib_main_t * vm,
+		 unformat_input_t * input,
+		 vlib_cli_command_t * cmd)
+{
+  u32 index;
+  int enable = 1;
+  int rv;
+  gmon_main_t *gm = &gmon_main;
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
+    if (unformat (input, "%d", &index))
+      ;
+    else if (unformat (input, "disable"))
+      enable = 0;
+    else
+      return clib_error_return (0, "unknown input `%U'",
+                                format_unformat_error, input);
+  }
+
+  rv = significant_error_enable_disable (gm, index, enable);
+
+  switch (rv)
+    {
+    case 0:
+      break;
+
+    default:
+      return clib_error_return 
+        (0, "significant_error_enable_disable returned %d", rv);
+    }
+
+  return 0;
+}
+
+VLIB_CLI_COMMAND (set_significant_error_command, static) = {
+    .path = "set significant error",
+    .short_help = "set significant error <counter-index-nnn>",
+    .function = set_significant_error_command_fn,
+};