Luajit API and some examples

Change-Id: Ia140c4750f06870c40b7058c4afb2e20ca633a49
Signed-off-by: Andrew Yourtchenko <ayourtch@gmail.com>

vpp-api/lua/README.md

@@ -0,0 +1,55 @@
+This is the experimental version of Lua API, aimed for the luajit use.
+
+Please take a look and send the feedback to ayourtch@gmail.com.
+
+To run the examples here:
+
+1) install luajit - "sudo apt-get install luajit" on ubuntu
+
+2) make build-vpp-api in the top directory
+
+3) "make" in this directory to build libcough.so
+
+4) "make run" in a separate terminal window
+
+5) sudo luajit examples/example-cli.lua
+
+This will result in something like this:
+
+libcough detected
+
+Version:        17.01-rc0~37-g8b3191e
+00000000  31 37 2E 30 31 2D 72 63  30 7E 33 37 2D 67 38 62  17.01-rc0~37-g8b
+00000010  33 31 39 31 65 00 00 00  00 00 00 00 00 00 00 00  3191e...........
+
+{ [1] = { ["luaapi_message_name"] = show_version_reply,["program"] = vpe,["version"] = 17.01-rc0~37-g8b3191e,["build_date"] = Fri Nov 11 15:30:21 UTC 2016,["retval"] = 0,["build_directory"] = /home/ubuntu/vpp,["_vl_msg_id"] = 166,["context"] = 0,} ,}
+---
+{ [1] = { ["luaapi_message_name"] = cli_inband_reply,["_vl_msg_id"] = 90,["length"] = 94,["reply"] = vpp v17.01-rc0~37-g8b3191e built by ubuntu on vpp-lapi-commit at Fri Nov 11 15:30:21 UTC 2016
+,["retval"] = 0,["context"] = 0,} ,}
+---
+
+6) You can also run the performance test bench:
+
+$ sudo luajit bench.lua
+libcough detected
+
+10001 iterations, average speed 4108LL per second
+10001 iterations, average speed 4660LL per second
+10001 iterations, average speed 4095LL per second
+10001 iterations, average speed 4542LL per second
+10001 iterations, average speed 8048LL per second
+10001 iterations, average speed 6805LL per second
+10001 iterations, average speed 5170LL per second
+10001 iterations, average speed 6585LL per second
+10001 iterations, average speed 6714LL per second
+10001 iterations, average speed 6942LL per second
+Average tps across the tests: 5766LL
+
+Note: the above is run in an lxd container running inside 2-core
+xhyve VM on a Macbook Pro, so I would not take the performance numbers for granted :)
+
+The "examples" directory contains a few naive examples, as well as a couple of more 
+advanced ones - a tab-completing CLI for VPP that can call both the APIs and CLI,
+and also a small test utility which I use for automating some small tests using
+VPP.
+

vpp-api/lua/bench.lua

@@ -0,0 +1,73 @@
+--[[
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+]]
+
+local vpp = require "vpp-lapi"
+
+local ffi = require "ffi"
+
+ffi.cdef([[
+   struct timespec {
+               long tv_sec;        /* seconds */
+               long tv_nsec;       /* nanoseconds */
+           };
+
+   int clock_gettime(int clk_id, struct timespec *tp);
+]])
+
+
+local time_cache = ffi.new("struct timespec[1]")
+local time_cache_1 = time_cache[0]
+function get_ns()
+  ffi.C.clock_gettime(0, time_cache)
+  return time_cache_1.tv_nsec + 1000000000 * time_cache_1.tv_sec
+end
+
+function do_bench()
+  local cycle_start = get_ns()
+  local n_iterations =  10000
+  local count = 1
+  for i = 1,n_iterations do
+    -- print(i)
+    vpp:api_call("show_version")
+    count = count + 1
+    -- print(i, "done")
+  end
+  cycle_end = get_ns()
+  local tps = n_iterations*1000000000LL/(cycle_end - cycle_start)
+  print (tostring(count) .. " iterations, average speed " .. tostring(tps) .. " per second")
+  return tps
+end
+
+root_dir = "/home/ubuntu/vpp"
+pneum_path = root_dir .. "/build-root/install-vpp_debug-native/vpp-api/lib64/libpneum.so"
+
+vpp:init({ pneum_path = pneum_path })
+
+vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vlib-api/vlibmemory/memclnt.api")
+vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vpp/vpp-api/vpe.api")
+
+vpp:connect("lua-bench")
+local n_tests = 10
+local tps_acc = 0LL
+for i=1,n_tests do
+  tps_acc = tps_acc + do_bench()
+end
+print("Average tps across the tests: " .. tostring(tps_acc/n_tests))
+
+vpp:disconnect()
+
+

vpp-api/lua/cough.c

@@ -0,0 +1,183 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* 
+ * This is a temporary helper library to seamlessly run against
+ * the current API as exposed by libpneum.
+ * In the future once the sync API is exposed as well as
+ * a way to free the received data, this can go away.
+ */
+
+#include <dlfcn.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <pthread.h>
+
+pthread_mutex_t mut;
+pthread_mutex_t *cb_lock = &mut;
+
+void *pneum_handle = NULL;
+
+typedef void* (*arbitrary)();
+
+int (*orig_pneum_connect)(char *name, char *chroot_prefix) = NULL;
+int (*orig_pneum_connect_sync)(char *name, char *chroot_prefix) = NULL;
+int (*orig_pneum_disconnect)(void) = NULL;
+int (*orig_pneum_read)(char **data, int *l) = NULL;
+int (*orig_pneum_write)(char *data, int len) = NULL;
+int (*orig_pneum_has_data)(void) = NULL;
+int (*orig_pneum_data_free)(char *data) = NULL;
+
+arbitrary my_function = NULL;
+
+typedef uint8_t u8;
+typedef uint32_t u32;
+
+u8 *cough_read_buffer;
+u32 cough_read_buffer_size = 1000000;
+u32 cough_read_head = 0;
+u32 cough_read_lock_start = 0; /* lock_start till head is busy memory */
+u32 cough_read_tail = 0;
+
+
+int wrap_pneum_callback(char *data, int len) {
+  // printf("Cough callback! with %d bytes\n", len);
+  pthread_mutex_lock(cb_lock);
+  if(cough_read_lock_start == cough_read_head) {
+    // printf("Reset read head!\n");
+    cough_read_head = 0;
+    cough_read_tail = 0;
+    cough_read_lock_start = 0;
+  }
+  u32 store_len = len;
+  memcpy(cough_read_buffer + cough_read_head, &store_len, sizeof(u32));
+  cough_read_head += sizeof(u32);
+  memcpy(cough_read_buffer + cough_read_head, data, len);
+  cough_read_head += len;
+  pthread_mutex_unlock(cb_lock);
+  return len;
+}
+
+int cough_pneum_attach(char *pneum_fname, char *cough_fname) {
+  /* this is needed to make the pneum aware of the wrap_pneum_callback */
+  pneum_handle = dlopen(cough_fname, RTLD_NOW|RTLD_GLOBAL);
+  /* now let's try to load pneum itself */
+  pneum_handle = dlopen(pneum_fname, RTLD_NOW|RTLD_GLOBAL);
+  if (pneum_handle) {
+    *(void**)(&orig_pneum_connect) = dlsym(pneum_handle,"pneum_connect");
+    *(void**)(&orig_pneum_connect_sync) = dlsym(pneum_handle,"pneum_connect_sync");
+    *(void**)(&orig_pneum_disconnect) = dlsym(pneum_handle,"pneum_disconnect");
+    *(void**)(&orig_pneum_read) = dlsym(pneum_handle,"pneum_read");
+    *(void**)(&orig_pneum_write) = dlsym(pneum_handle,"pneum_write");
+    *(void**)(&orig_pneum_has_data) = dlsym(pneum_handle,"pneum_has_data");
+    *(void**)(&orig_pneum_data_free) = dlsym(pneum_handle,"pneum_data_free");
+    // If you uncomment the below line we pretend we have an async-only libpneum
+    orig_pneum_connect_sync = NULL;
+    cough_read_buffer = malloc(cough_read_buffer_size);
+  } else {
+    printf("Could not get cough handle\n");
+    printf("Error: %s", dlerror());
+    return -1;
+  }
+
+  *(void**)(&my_function) = dlsym(pneum_handle,"something");
+}
+
+
+int pneum_connect(char *name, char *chroot_prefix) {
+  if(orig_pneum_connect) {
+    return(orig_pneum_connect(name, chroot_prefix));
+  } else {
+    printf("COUGH: pneum_connect\n");
+    return -1;
+  }
+}
+int pneum_connect_sync(char *name, char *chroot_prefix) {
+  if(orig_pneum_connect_sync) {
+    int ret = (orig_pneum_connect_sync(name, chroot_prefix));
+    return ret;
+  } else {
+    return(orig_pneum_connect(name, chroot_prefix));
+  }
+}
+
+
+int pneum_disconnect(void) {
+  if(orig_pneum_disconnect) {
+    return orig_pneum_disconnect();
+  } else {
+    printf("COUGH: pneum_disconnect\n");
+    return -1;
+  }
+}
+
+int pneum_has_data(void) {
+  if (orig_pneum_connect_sync) {
+    /* always return 1 in a pass-through case */
+    return 1;
+  } else {
+    // printf("COUGH: pneum_has_data\n");
+    return (cough_read_head != cough_read_tail);
+  }
+}
+
+
+int pneum_read(char **data, int *l) {
+  if(orig_pneum_connect_sync) {
+    return orig_pneum_read(data, l);
+  } else { 
+    while(!pneum_has_data());
+    u32 n_bytes;
+    pthread_mutex_lock(cb_lock);
+    memcpy(&n_bytes, cough_read_buffer + cough_read_tail, sizeof(u32));
+    cough_read_tail += sizeof(u32);
+    void * dataptr = (void *) (cough_read_buffer + cough_read_tail);
+    *data = dataptr;
+    cough_read_tail += n_bytes;
+    *l = n_bytes;
+    pthread_mutex_unlock(cb_lock);
+    return n_bytes; 
+  }
+}
+
+int pneum_write(char *data, int len) {
+  if(orig_pneum_write) {
+    return(orig_pneum_write(data, len));
+  } else {
+    printf("COUGH: pneum_write\n");
+    return -1;
+  }
+}
+
+void pneum_data_free(char *data) {
+  if (orig_pneum_connect_sync) {
+    if(orig_pneum_data_free) {
+      orig_pneum_data_free(data);
+    } else {
+      printf("COUGH: pneum_data_free\n");
+    }
+  } else {
+    u32 *len;
+    uint32_t index = ((u8*)data) - cough_read_buffer;
+    pthread_mutex_lock(cb_lock);
+    if ((index < cough_read_head) && (index > cough_read_lock_start)) {
+      len = (void *)(data - sizeof(u32));
+      cough_read_lock_start = index + *len;
+    }
+    pthread_mutex_unlock(cb_lock);
+  }
+}

vpp-api/lua/examples/cli/README.md

@@ -0,0 +1,5 @@
+This is a small experiment to have a wrapper CLI which can call both API functions as well as debug CLI.
+
+To facilitate tab completion and help, the API call names are broken up with spaces replacing the underscores.
+
+

vpp-api/lua/examples/example-acl-plugin.lua

@@ -0,0 +1,110 @@
+--[[
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+]]
+
+
+vpp = require "vpp-lapi"
+
+root_dir = "/home/ubuntu/vpp"
+pneum_path = root_dir .. "/build-root/install-vpp_debug-native/vpp-api/lib64/libpneum.so"
+
+vpp:init({ pneum_path = pneum_path })
+
+vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vlib-api/vlibmemory/memclnt.api")
+vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vpp/vpp-api/vpe.api")
+vpp:connect("aytest")
+vpp:consume_api(root_dir .. "/plugins/acl-plugin/acl/acl.api", "acl")
+
+-- api calls
+reply = vpp:api_call("show_version")
+print("Version: ", reply[1].version)
+print(vpp.hex_dump(reply[1].version))
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_del", { context = 42, acl_index = 230 })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_del", { context = 42, acl_index = 8 })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_del", { context = 42, acl_index = 15 })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_add", { context = 42, count = 2, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 } } })
+print(vpp.dump(reply))
+print("---")
+interface_acl_in = reply[1].acl_index
+
+reply = vpp:api_call("acl_add", { context = 42, count = 3, r = { { is_permit = 1, is_ipv6 = 1 }, { is_permit = 0, is_ipv6 = 1 }, { is_permit = 1, is_ipv6 = 0 } } })
+print(vpp.dump(reply))
+print("---")
+interface_acl_out = reply[1].acl_index
+
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 1, acl_index = interface_acl_in })
+print(vpp.dump(reply))
+print("---")
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 1, acl_index = interface_acl_in })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 0, acl_index = interface_acl_out })
+print(vpp.dump(reply))
+print("---")
+reply = vpp:api_call("acl_interface_add_del", { context = 42, sw_if_index = 0, is_add = 1, is_input = 0, acl_index = interface_acl_out })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_add", { context = 42, count = 0 })
+print(vpp.dump(reply))
+print("---")
+
+acl_index_to_delete = reply[1].acl_index
+print("Deleting " .. tostring(acl_index_to_delete))
+reply = vpp:api_call("acl_del", { context = 42, acl_index = acl_index_to_delete })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 0})
+for ri, rv in ipairs(reply) do 
+  print("Reply message #" .. tostring(ri))
+  print(vpp.dump(rv))
+  for ai, av in ipairs(rv.r) do
+    print("ACL rule #" .. tostring(ai) .. " : " .. vpp.dump(av))
+  end
+   
+end
+print("---")
+
+reply = vpp:api_call("acl_del", { context = 42, acl_index = interface_acl_out })
+print(vpp.dump(reply))
+print("---")
+reply = vpp:api_call("acl_del", { context = 42, acl_index = interface_acl_in })
+print(vpp.dump(reply))
+print("---")
+
+reply = vpp:api_call("acl_dump", { context = 42, sw_if_index = 0})
+print(vpp.dump(reply))
+print("---")
+
+
+vpp:disconnect()
+
+

vpp-api/lua/examples/example-classifier.lua

@@ -0,0 +1,51 @@
+--[[
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+]]
+
+
+local vpp = require "vpp-lapi"
+local bit = require("bit")
+
+root_dir = "/home/ubuntu/vpp"
+pneum_path = root_dir .. "/build-root/install-vpp_debug-native/vpp-api/lib64/libpneum.so"
+
+vpp:init({ pneum_path = pneum_path })
+
+vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vlib-api/vlibmemory/memclnt.api")
+vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vpp/vpp-api/vpe.api")
+
+vpp:connect("aytest")
+
+-- api calls
+
+print("Calling API to add a new classifier table")
+reply = vpp:api_call("classify_add_del_table", {
+  context = 43,
+  memory_size = bit.lshift(2, 20),
+  client_index = 42,
+  is_add = 1,
+  nbuckets = 32,
+  skip_n_vectors = 0,
+  match_n_vectors = 1,
+  mask = "\255\255\255\255\255\255\255\255" .. "\255\255\255\255\255\255\255\255"
+})
+print(vpp.dump(reply))
+print("---")
+
+
+vpp:disconnect()
+
+

vpp-api/lua/examples/example-cli.lua

@@ -0,0 +1,45 @@
+--[[
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+]]
+
+vpp = require "vpp-lapi"
+
+root_dir = "/home/ubuntu/vpp"
+pneum_path = root_dir .. "/build-root/install-vpp_debug-native/vpp-api/lib64/libpneum.so"
+
+vpp:init({ pneum_path = pneum_path })
+
+vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vlib-api/vlibmemory/memclnt.api")
+vpp:consume_api(root_dir .. "/build-root/install-vpp_debug-native/vpp/vpp-api/vpe.api")
+
+vpp:connect("aytest")
+
+-- api calls
+reply = vpp:api_call("show_version")
+print("Version: ", reply[1].version)
+print(vpp.hex_dump(reply[1].version))
+print(vpp.dump(reply))
+print("---")
+
+
+reply = vpp:api_call("cli_inband", { cmd = "show vers" }) 
+print(vpp.dump(reply))
+print("---")
+
+
+vpp:disconnect()
+
+

vpp-api/lua/examples/lute/README.md

@@ -0,0 +1,66 @@
+LUTE: Lua Unit Test Environment
+
+This is a small helper utility to automate some simple tests
+that one might need to do.
+
+Think of it as a hybrid of a screen and expect who
+also took some habits from HTML inline code.
+
+It is quite probably useless for building anything serious,
+but practice shows it is quite efficient at allowing
+convenient temporary quick tests, and for something
+that was written over a course of a couple of evenings it
+is quite a nice little helper tool.
+
+It allows do launch and drive multiple shell sessions,
+and by virtue of having been written in Lua, it of course
+also allows to add the business logic using the Lua code.
+
+If you launch the lute without parameters, it gives you
+the interactive shell to execute the commands in.
+
+If you launch it with an argument, it will attempt to
+read and execute the commands from the file.
+
+Commands:
+
+shell FOO
+
+  spawn a shell in a new PTY under the label FOO.
+
+run FOO bar
+
+  Send "bar" keystrokes followed by "ENTER" to the session FOO
+
+  Special case: "break" word on its own gets translated into ^C being sent.
+
+cd FOO
+
+  "change domain" into session FOO. All subsequent inputs will go,
+  line-buffered, into the session FOO. To jump back up, use ^D (Control-D),
+  or within the file, use ^D^D^D (caret D caret D caret D on its own line)
+
+expect FOO blablabla
+
+  Pause further interpretation of the batch mode until you see "blablabla"
+  in the output of session FOO, or until timeout happens.
+
+sleep N
+
+  Sleep an integer N seconds, if you are in batch mode.
+
+echo blabla
+
+  Echo the remainder of the line to standard output.
+
+For Lua code, there is a pre-existing pseudo-session called "lua",
+which accepts "run lua" command which does what you would expect
+(evaluate the rest of the string in Lua context - being the same
+as lute itself). Also you can do "cd lua" and get into a
+multiline-enabled interpreter shell.
+
+This way for the VPP case you can automate some of the things in your routine
+that you would have to have done manually, and test drive API as well
+as use the realistic native OS components to create the environment around it.
+
+

vpp-api/lua/examples/lute/script.lute

@@ -0,0 +1,7 @@
+shell s1
+expect s1 $
+run s1 echo testing123
+expect s1 $
+run s1 echo done
+quit
+