api: vppapitrace JSON/API trace converter

Usage: vppapitrace.py [-h] [--debug] [--apidir APIDIR] {convert,replay} ...

optional arguments:
  -h, --help        show this help message and exit
  --debug           enable debug mode
  --apidir APIDIR   Location of JSON API definitions

subcommands:
  valid subcommands

  {convert,replay}  additional help
    convert         Convert API trace to JSON or Python and back
    replay          Replay messages to running VPP instance

To convert an API trace file to JSON:
vppapitrace convert /tmp/api.trace trace.json

To convert an (edited) JSON file back to API trace for replay:
vppapitrace convert trace.json api-edited.trace

To generate a Python file that can be replayed:
vppapitrace convert /tmp/api.trace trace.py
vppapitrace convert trace.json trace.py

Replay it to a running VPP instance:
vppapitrace replay --socket /tmp/api.trace

In VPP that file can be replayed with:
vpp# api trace replay api-edited.trace

This patch also modifies the API binary trace format, to include the
message id to message name table.

Ticket: VPP-1733
Change-Id: Ie6441efb53c1c93c9f778f6ae9c1758bccc8dd87
Type: refactor
Signed-off-by: Ole Troan <ot@cisco.com>
(cherry picked from commit edfe2c0079a756f5fb1108037c39450e3521c8bd)
Signed-off-by: Andrew Yourtchenko <ayourtch@gmail.com>

MAINTAINERS

@@ -435,6 +435,11 @@ I:	vppapigen
 M:	Ole Troan <otroan@employees.org>
 F:	src/tools/vppapigen/
 
+API trace tool
+I:	vppapitrace
+M:	Ole Troan <otroan@employees.org>
+F:	src/tools/vppapitrace/
+
 Binary API Compiler for C and C++
 I:	vapi
 M:	Ole Troan <ot@cisco.com>

src/tools/vppapitrace/vppapitrace

@@ -0,0 +1 @@
+vppapitrace.py

src/vlibapi/api.h

@@ -30,9 +30,9 @@
 
 /* *INDENT-OFF* */
 typedef CLIB_PACKED ( struct {
-   u8 endian;
-   u8 wrapped;
    u32 nitems;
+   u32 msgtbl_size;
+   u8 wrapped;
 }) vl_api_trace_file_header_t;
 /* *INDENT-ON* */
 

src/vlibapi/api_shared.c

@@ -189,6 +189,29 @@ vl_msg_api_trace_free (api_main_t * am, vl_api_trace_which_t which)
   return 0;
 }
 
+u8 *
+vl_api_serialize_message_table (api_main_t * am, u8 * vector)
+{
+  serialize_main_t _sm, *sm = &_sm;
+  hash_pair_t *hp;
+  u32 nmsg = hash_elts (am->msg_index_by_name_and_crc);
+
+  serialize_open_vector (sm, vector);
+
+  /* serialize the count */
+  serialize_integer (sm, nmsg, sizeof (u32));
+
+  /* *INDENT-OFF* */
+  hash_foreach_pair (hp, am->msg_index_by_name_and_crc,
+  ({
+    serialize_likely_small_unsigned_integer (sm, hp->value[0]);
+    serialize_cstring (sm, (char *) hp->key);
+  }));
+  /* *INDENT-ON* */
+
+  return serialize_close_vector (sm);
+}
+
 int
 vl_msg_api_trace_save (api_main_t * am, vl_api_trace_which_t which, FILE * fp)
 {
@@ -223,15 +246,24 @@ vl_msg_api_trace_save (api_main_t * am, vl_api_trace_which_t which, FILE * fp)
     }
 
   /* Write the file header */
-  fh.nitems = vec_len (tp->traces);
-  fh.endian = tp->endian;
   fh.wrapped = tp->wrapped;
+  fh.nitems = clib_host_to_net_u32 (vec_len (tp->traces));
+  u8 *m = vl_api_serialize_message_table (am, 0);
+  clib_warning ("Message table length %d", vec_len (m));
+  fh.msgtbl_size = clib_host_to_net_u32 (vec_len (m));
 
   if (fwrite (&fh, sizeof (fh), 1, fp) != 1)
     {
       return (-10);
     }
 
+  /* Write the message table */
+  if (fwrite (m, vec_len (m), 1, fp) != 1)
+    {
+      return (-14);
+    }
+  vec_free (m);
+
   /* No-wrap case */
   if (tp->wrapped == 0)
     {

src/vlibmemory/socket_api.c

@@ -692,9 +692,9 @@ reply:
 }
 
 #define foreach_vlib_api_msg                    	\
-_(SOCKCLNT_CREATE, sockclnt_create)             	\
+  _(SOCKCLNT_CREATE, sockclnt_create, 1)             	\
-_(SOCKCLNT_DELETE, sockclnt_delete)			\
+  _(SOCKCLNT_DELETE, sockclnt_delete, 1)		\
-_(SOCK_INIT_SHM, sock_init_shm)
+  _(SOCK_INIT_SHM, sock_init_shm, 1)
 
 clib_error_t *
 vl_sock_api_init (vlib_main_t * vm)
@@ -710,13 +710,13 @@ vl_sock_api_init (vlib_main_t * vm)
   if (sm->socket_name == 0)
     return 0;
 
-#define _(N,n)                                                  \
+#define _(N,n,t)						\
     vl_msg_api_set_handlers(VL_API_##N, #n,                     \
                            vl_api_##n##_t_handler,              \
                            vl_noop_handler,                     \
                            vl_api_##n##_t_endian,               \
                            vl_api_##n##_t_print,                \
-                           sizeof(vl_api_##n##_t), 1);
+                           sizeof(vl_api_##n##_t), t);
   foreach_vlib_api_msg;
 #undef _
 

src/vlibmemory/vlib_api.c

@@ -68,29 +68,6 @@ vl_api_trace_plugin_msg_ids_t_print (vl_api_trace_plugin_msg_ids_t * a,
 #include <vlibmemory/vl_memory_api_h.h>
 #undef vl_endianfun
 
-u8 *
-vl_api_serialize_message_table (api_main_t * am, u8 * vector)
-{
-  serialize_main_t _sm, *sm = &_sm;
-  hash_pair_t *hp;
-  u32 nmsg = hash_elts (am->msg_index_by_name_and_crc);
-
-  serialize_open_vector (sm, vector);
-
-  /* serialize the count */
-  serialize_integer (sm, nmsg, sizeof (u32));
-
-  /* *INDENT-OFF* */
-  hash_foreach_pair (hp, am->msg_index_by_name_and_crc,
-  ({
-    serialize_likely_small_unsigned_integer (sm, hp->value[0]);
-    serialize_cstring (sm, (char *) hp->key);
-  }));
-  /* *INDENT-ON* */
-
-  return serialize_close_vector (sm);
-}
-
 static void
 vl_api_get_first_msg_id_t_handler (vl_api_get_first_msg_id_t * mp)
 {

src/vlibmemory/vlib_api_cli.c

@@ -402,10 +402,9 @@ vl_msg_api_process_file (vlib_main_t * vm, u8 * filename,
   struct stat statb;
   size_t file_size;
   u8 *msg;
-  u8 endian_swap_needed = 0;
   api_main_t *am = &api_main;
   u8 *tmpbuf = 0;
-  u32 nitems;
+  u32 nitems, nitems_msgtbl;
   void **saved_print_handlers = 0;
 
   fd = open ((char *) filename, O_RDONLY);
@@ -443,14 +442,7 @@ vl_msg_api_process_file (vlib_main_t * vm, u8 * filename,
     }
   close (fd);
 
-  if ((clib_arch_is_little_endian && hp->endian == VL_API_BIG_ENDIAN)
+  nitems = ntohl (hp->nitems);
-      || (clib_arch_is_big_endian && hp->endian == VL_API_LITTLE_ENDIAN))
-    endian_swap_needed = 1;
-
-  if (endian_swap_needed)
-    nitems = ntohl (hp->nitems);
-  else
-    nitems = hp->nitems;
 
   if (last_index == (u32) ~ 0)
     {
@@ -473,10 +465,27 @@ vl_msg_api_process_file (vlib_main_t * vm, u8 * filename,
       saved_print_handlers = (void **) vec_dup (am->msg_print_handlers);
       vl_msg_api_custom_dump_configure (am);
     }
-
-
   msg = (u8 *) (hp + 1);
 
+  u16 *msgid_vec = 0;
+  serialize_main_t _sm, *sm = &_sm;
+  u32 msgtbl_size = ntohl (hp->msgtbl_size);
+  u8 *name_and_crc;
+
+  unserialize_open_data (sm, msg, msgtbl_size);
+  unserialize_integer (sm, &nitems_msgtbl, sizeof (u32));
+
+  for (i = 0; i < nitems_msgtbl; i++)
+    {
+      u16 msg_index = unserialize_likely_small_unsigned_integer (sm);
+      unserialize_cstring (sm, (char **) &name_and_crc);
+      u16 msg_index2 = vl_msg_api_get_msg_index (name_and_crc);
+      vec_validate (msgid_vec, msg_index);
+      msgid_vec[msg_index] = msg_index2;
+    }
+
+  msg += msgtbl_size;
+
   for (i = 0; i < first_index; i++)
     {
       trace_cfg_t *cfgp;
@@ -486,11 +495,9 @@ vl_msg_api_process_file (vlib_main_t * vm, u8 * filename,
       size = clib_host_to_net_u32 (*(u32 *) msg);
       msg += sizeof (u32);
 
-      if (clib_arch_is_little_endian)
+      msg_id = ntohs (*((u16 *) msg));
-	msg_id = ntohs (*((u16 *) msg));
+      if (msg_id < vec_len (msgid_vec))
-      else
+	msg_id = msgid_vec[msg_id];
-	msg_id = *((u16 *) msg);
-
       cfgp = am->api_trace_cfg + msg_id;
       if (!cfgp)
 	{
@@ -507,7 +514,6 @@ vl_msg_api_process_file (vlib_main_t * vm, u8 * filename,
   for (; i <= last_index; i++)
     {
       trace_cfg_t *cfgp;
-      u16 *msg_idp;
       u16 msg_id;
       int size;
 
@@ -517,10 +523,11 @@ vl_msg_api_process_file (vlib_main_t * vm, u8 * filename,
       size = clib_host_to_net_u32 (*(u32 *) msg);
       msg += sizeof (u32);
 
-      if (clib_arch_is_little_endian)
+      msg_id = ntohs (*((u16 *) msg));
-	msg_id = ntohs (*((u16 *) msg));
+      if (msg_id < vec_len (msgid_vec))
-      else
+	{
-	msg_id = *((u16 *) msg);
+	  msg_id = msgid_vec[msg_id];
+	}
 
       cfgp = am->api_trace_cfg + msg_id;
       if (!cfgp)
@@ -538,12 +545,10 @@ vl_msg_api_process_file (vlib_main_t * vm, u8 * filename,
       clib_memset (tmpbuf, 0xf, sizeof (uword));
 
       /*
-       * Endian swap if needed. All msg data is supposed to be
+       * Endian swap if needed. All msg data is supposed to be in
-       * in network byte order. All msg handlers are supposed to
+       * network byte order.
-       * know that. The generic message dumpers don't know that.
-       * One could fix apigen, I suppose.
        */
-      if ((which == DUMP && clib_arch_is_little_endian) || endian_swap_needed)
+      if ((which == DUMP && clib_arch_is_little_endian))
 	{
 	  void (*endian_fp) (void *);
 	  if (msg_id >= vec_len (am->msg_endian_handlers)
@@ -562,7 +567,7 @@ vl_msg_api_process_file (vlib_main_t * vm, u8 * filename,
       /* msg_id always in network byte order */
       if (clib_arch_is_little_endian)
 	{
-	  msg_idp = (u16 *) (tmpbuf + sizeof (uword));
+	  u16 *msg_idp = (u16 *) (tmpbuf + sizeof (uword));
 	  *msg_idp = msg_id;
 	}
 
@@ -1051,7 +1056,7 @@ dump_api_table_file_command_fn (vlib_main_t * vm,
       item->crc = extract_crc (name_and_crc);
       item->which = 0;		/* file */
     }
-  serialize_close (sm);
+  unserialize_close (sm);
 
   /* Compare with the current image? */
   if (compare_current)

src/vnet/ip/ip_api.c

@@ -2545,7 +2545,7 @@ vl_api_ip_probe_neighbor_t_handler (vl_api_ip_probe_neighbor_t * mp)
 
   BAD_SW_IF_INDEX_LABEL;
 
-  REPLY_MACRO (VL_API_PROXY_ARP_INTFC_ENABLE_DISABLE_REPLY);
+  REPLY_MACRO (VL_API_IP_PROBE_NEIGHBOR_REPLY);
 }
 
 static void

src/vpp-api/python/vpp_papi/__init__.py

@@ -2,6 +2,7 @@ from .vpp_papi import FuncWrapper, VPP, VppApiDynamicMethodHolder  # noqa: F401
 from .vpp_papi import VppEnum, VppEnumType  # noqa: F401
 from .vpp_papi import VPPIOError, VPPRuntimeError, VPPValueError  # noqa: F401
 from .vpp_papi import VPPApiClient  # noqa: F401
+from .vpp_papi import VPPApiJSONFiles # noqa: F401
 from . macaddress import MACAddress, mac_pton, mac_ntop  # noqa: F401
 
 # sorted lexicographically