vlib: reset stop_timer_handle on expired processes

Type: fix

The main loop populates a vector of suspended process nodes to dispatch
by calling TW (tw_timer_expire_timers_vec), which identifies expired
timers and appends the user handle for each one to the vector.

Subsequently, the vector is iterated and the process node corresponding
to each handle is dispatched. The vast majority of the time, the process
node will end up suspending itself again to wait for a new timer or
event.

Given a process node A whose timer has expired, between the point when
the timer expired and the point when A is dispatched and suspends itself
again, its stop_timer_handle contains a stale value.

If another process node B is dispatched before A is dispatched, it may
end up using the timer ID that A formerly used. If another process node
C is dispatched after B and before A and calls
vlib_process_signal_event() to signal A, the timer started by B can be
deleted by vlib_process_signal_event_helper().

After getting the vector of process node IDs for expired timers, reset
the stop_timer_handle on each of those nodes.

Change-Id: I266da438e76e1fc356016da0b9b4941efac1c28a
Signed-off-by: Matthew Smith <mgsmith@netgate.com>
(cherry picked from commit 9aa4ac55b0a205e333e567f87c3cc9379af2363c)

.gitreview

@@ -2,3 +2,4 @@
 host=gerrit.fd.io
 port=29418
 project=vpp
+defaultbranch=stable/2302

docs/aboutvpp/releasenotes/index.rst

@@ -6,7 +6,10 @@ Release notes
 .. toctree::
     :maxdepth: 2
 
+    v23.02
+    v22.10.1
     v22.10
+    v22.06.1
     v22.06
     v22.02
     past

docs/aboutvpp/releasenotes/past.rst

@@ -39,4 +39,3 @@ Past releases
     v17.01
     v16.09
     v16.06
-

docs/aboutvpp/releasenotes/v22.06.1.rst

@@ -0,0 +1,12 @@
+Release notes for VPP 22.06.1
+=============================
+
+This is bug fix release.
+
+Of particular importance, this release contains the fix for
+`JIRA VPP-2307: CVE-2022-46397 FD.io VPP (Vector Packet Processor) IPSec generates a predictable IV in AES-CBC mode <https://jira.fd.io/browse/VPP-2037>`__
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/2206>`__

docs/aboutvpp/releasenotes/v22.10.1.rst

@@ -0,0 +1,12 @@
+Release notes for VPP 22.10.1
+=============================
+
+This is bug fix release.
+
+Of particular importance, this release contains the fix for
+`JIRA VPP-2307: CVE-2022-46397 FD.io VPP (Vector Packet Processor) IPSec generates a predictable IV in AES-CBC mode <https://jira.fd.io/browse/VPP-2037>`__
+
+For the full list of fixed issues please refer to:
+
+- fd.io `JIRA <https://jira.fd.io>`__
+- git `commit log <https://git.fd.io/vpp/log/?h=stable/2210>`__

extras/packetforge/parsegraph/edges/ipv6_after_gtppsc.json

@@ -0,0 +1,5 @@
+{
+    "type" : "edge",
+    "start" : "gtppsc",
+    "end" : "ipv6"
+}

extras/packetforge/parsegraph/nodes/mac.json

@@ -3,13 +3,13 @@
     "name" : "mac",
     "layout" : [
         {
-            "name" : "src",
+            "name" : "dst",
             "size" : "48",
             "format" : "mac",
             "default" : "00:00:00:00:00:01"
         },
         {
-            "name" : "dst",
+            "name" : "src",
             "size" : "48",
             "format" : "mac",
             "default" : "00:00:00:00:00:02"

src/plugins/avf/avf_fdir_lib.c

@@ -28,7 +28,7 @@
 static inline int
 fls_u32 (u32 x)
 {
-  return (x == 0) ? 0 : 32 - count_leading_zeros (x);
+  return (x == 0) ? 0 : 64 - count_leading_zeros (x);
 }
 
 static inline int
@@ -257,9 +257,9 @@ avf_fdir_rcfg_set_field (struct avf_fdir_conf *rcfg, int layer,
 	    {
 	      rcfg->input_set |= AVF_INSET_IPV6_DST;
 	      VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT (hdr, IPV6, DST);
-
-	      clib_memcpy (hdr->buffer, ipv6_spec, sizeof (*ipv6_spec));
 	    }
+
+	  clib_memcpy (hdr->buffer, ipv6_spec, sizeof (*ipv6_spec));
 	}
 
       break;

src/plugins/avf/device.c

@@ -1763,7 +1763,7 @@ avf_create_if (vlib_main_t * vm, avf_create_if_args_t * args)
   /* set hw interface caps */
   vnet_hw_if_set_caps (vnm, ad->hw_if_index,
 		       VNET_HW_IF_CAP_INT_MODE | VNET_HW_IF_CAP_MAC_FILTER |
-			 VNET_HW_IF_CAP_L4_TX_CKSUM | VNET_HW_IF_CAP_TCP_GSO);
+			 VNET_HW_IF_CAP_TX_CKSUM | VNET_HW_IF_CAP_TCP_GSO);
 
   for (i = 0; i < ad->n_rx_queues; i++)
     {

src/plugins/avf/flow.c

@@ -506,6 +506,7 @@ pattern_end:
   if (f->actions & VNET_FLOW_ACTION_RSS)
     {
       avf_actions[action_count].conf = &act_rss;
+      is_fdir = false;
 
       if ((act_rss.func = avf_flow_convert_rss_func (f->rss_fun)) ==
 	  AVF_ETH_HASH_FUNCTION_MAX)
@@ -522,8 +523,6 @@ pattern_end:
 	  is_fdir = true;
 	}
 
-      is_fdir = false;
-
       if (fate == true)
 	{
 	  rv = VNET_FLOW_ERROR_INTERNAL;

src/plugins/avf/output.c

@@ -123,8 +123,6 @@ avf_tx_prepare_cksum (vlib_buffer_t * b, u8 is_tso)
 	  sum = ~clib_ip_csum ((u8 *) &psh, sizeof (psh));
 	}
 
-  /* ip_csum does a byte swap for some reason... */
-  sum = clib_net_to_host_u16 (sum);
   if (is_tcp)
     tcp->checksum = sum;
   else

src/plugins/dpdk/device/flow.c

@@ -319,7 +319,8 @@ dpdk_flow_add (dpdk_device_t * xd, vnet_flow_t * f, dpdk_flow_entry_t * fe)
 
       if ((ip6_ptr->src_addr.mask.as_u64[0] == 0) &&
 	  (ip6_ptr->src_addr.mask.as_u64[1] == 0) &&
-	  (!ip6_ptr->protocol.mask))
+	  (ip6_ptr->dst_addr.mask.as_u64[0] == 0) &&
+	  (ip6_ptr->dst_addr.mask.as_u64[1] == 0) && (!ip6_ptr->protocol.mask))
 	{
 	  item->spec = NULL;
 	  item->mask = NULL;

src/plugins/nat/nat44-ed/nat44_ed_api.c

@@ -1202,7 +1202,7 @@ vl_api_nat44_ed_add_del_vrf_table_t_handler (
   vl_api_nat44_ed_add_del_vrf_table_reply_t *rmp;
   int rv = nat44_ed_add_del_vrf_table (clib_net_to_host_u32 (mp->table_vrf_id),
 				       mp->is_add);
-  REPLY_MACRO (VL_API_NAT44_ED_ADD_DEL_VRF_TABLE);
+  REPLY_MACRO (VL_API_NAT44_ED_ADD_DEL_VRF_TABLE_REPLY);
 }
 
 static void
@@ -1214,7 +1214,7 @@ vl_api_nat44_ed_add_del_vrf_route_t_handler (
   int rv =
     nat44_ed_add_del_vrf_route (clib_net_to_host_u32 (mp->table_vrf_id),
 				clib_net_to_host_u32 (mp->vrf_id), mp->is_add);
-  REPLY_MACRO (VL_API_NAT44_ED_ADD_DEL_VRF_ROUTE);
+  REPLY_MACRO (VL_API_NAT44_ED_ADD_DEL_VRF_ROUTE_REPLY);
 }
 
 static void

src/plugins/tlsopenssl/tls_openssl.c

@@ -186,18 +186,20 @@ openssl_read_from_ssl_into_fifo (svm_fifo_t * f, SSL * ssl)
       return 0;
     }
 
-  for (i = 1; i < n_fs; i++)
+  if (read == (int) fs[0].len)
     {
-      rv = SSL_read (ssl, fs[i].data, fs[i].len);
-      read += rv > 0 ? rv : 0;
-
-      if (rv < (int) fs[i].len)
+      for (i = 1; i < n_fs; i++)
 	{
-	  ossl_check_err_is_fatal (ssl, rv);
-	  break;
+	  rv = SSL_read (ssl, fs[i].data, fs[i].len);
+	  read += rv > 0 ? rv : 0;
+
+	  if (rv < (int) fs[i].len)
+	    {
+	      ossl_check_err_is_fatal (ssl, rv);
+	      break;
+	    }
 	}
     }
-
   svm_fifo_enqueue_nocopy (f, read);
 
   return read;

src/tools/vppapigen/vppapigen.py

@@ -752,6 +752,7 @@ class VPPAPIParser:
         """define : DEFINE ID '{' block_statements_opt '}' ';'"""
         self.fields = []
         p[0] = Define(p[2], [], p[4], self.last_comment)
+        self.last_comment = None
 
     def p_define_flist(self, p):
         """define : flist DEFINE ID '{' block_statements_opt '}' ';'"""
@@ -763,6 +764,7 @@ class VPPAPIParser:
             )
         else:
             p[0] = Define(p[3], p[1], p[5], self.last_comment)
+            self.last_comment = None
 
     def p_flist(self, p):
         """flist : flag

src/tools/vppapigen/vppapigen_json.py

@@ -77,7 +77,8 @@ def walk_defs(s, is_message=False):
             c = {}
             c["crc"] = "{0:#0{1}x}".format(t.crc, 10)
             c["options"] = t.options
-            c["comment"] = t.comment
+            if t.comment:
+                c["comment"] = t.comment
             d.append(c)
 
         r.append(d)

src/vcl/CMakeLists.txt

@@ -30,6 +30,11 @@ add_vpp_library(vppcom
   api_headers
 )
 
+option(LDP_HAS_GNU_SOURCE "LDP configured to use _GNU_SOURCE" ON)
+if (LDP_HAS_GNU_SOURCE)
+  add_compile_definitions(HAVE_GNU_SOURCE)
+endif(LDP_HAS_GNU_SOURCE)
+
 add_vpp_library(vcl_ldpreload
   SOURCES
   ldp_socket_wrapper.c

src/vcl/ldp.c

@@ -12,6 +12,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
+#ifdef HAVE_GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
 #include <unistd.h>
 #include <stdio.h>
 #include <signal.h>
@@ -52,6 +57,12 @@
 
 #define LDP_MAX_NWORKERS 32
 
+#ifdef HAVE_GNU_SOURCE
+#define SOCKADDR_GET_SA(__addr) __addr.__sockaddr__;
+#else
+#define SOCKADDR_GET_SA(__addr) _addr;
+#endif
+
 typedef struct ldp_worker_ctx_
 {
   u8 *io_buffer;
@@ -1052,8 +1063,9 @@ socketpair (int domain, int type, int protocol, int fds[2])
 }
 
 int
-bind (int fd, __CONST_SOCKADDR_ARG addr, socklen_t len)
+bind (int fd, __CONST_SOCKADDR_ARG _addr, socklen_t len)
 {
+  const struct sockaddr *addr = SOCKADDR_GET_SA (_addr);
   vls_handle_t vlsh;
   int rv;
 
@@ -1124,11 +1136,10 @@ done:
 }
 
 static inline int
-ldp_copy_ep_to_sockaddr (__SOCKADDR_ARG addr, socklen_t * __restrict len,
-			 vppcom_endpt_t * ep)
+ldp_copy_ep_to_sockaddr (struct sockaddr *addr, socklen_t *__restrict len,
+			 vppcom_endpt_t *ep)
 {
-  int rv = 0;
-  int sa_len, copy_len;
+  int rv = 0, sa_len, copy_len;
 
   ldp_init_check ();
 
@@ -1169,8 +1180,9 @@ ldp_copy_ep_to_sockaddr (__SOCKADDR_ARG addr, socklen_t * __restrict len,
 }
 
 int
-getsockname (int fd, __SOCKADDR_ARG addr, socklen_t * __restrict len)
+getsockname (int fd, __SOCKADDR_ARG _addr, socklen_t *__restrict len)
 {
+  struct sockaddr *addr = SOCKADDR_GET_SA (_addr);
   vls_handle_t vlsh;
   int rv;
 
@@ -1203,15 +1215,16 @@ getsockname (int fd, __SOCKADDR_ARG addr, socklen_t * __restrict len)
     }
   else
     {
-      rv = libc_getsockname (fd, addr, len);
+      rv = libc_getsockname (fd, _addr, len);
     }
 
   return rv;
 }
 
 int
-connect (int fd, __CONST_SOCKADDR_ARG addr, socklen_t len)
+connect (int fd, __CONST_SOCKADDR_ARG _addr, socklen_t len)
 {
+  const struct sockaddr *addr = SOCKADDR_GET_SA (_addr);
   vls_handle_t vlsh;
   int rv;
 
@@ -1291,8 +1304,9 @@ done:
 }
 
 int
-getpeername (int fd, __SOCKADDR_ARG addr, socklen_t * __restrict len)
+getpeername (int fd, __SOCKADDR_ARG _addr, socklen_t *__restrict len)
 {
+  struct sockaddr *addr = SOCKADDR_GET_SA (_addr);
   vls_handle_t vlsh;
   int rv;
 
@@ -1543,8 +1557,9 @@ __recv_chk (int fd, void *buf, size_t n, size_t buflen, int flags)
 static inline int
 ldp_vls_sendo (vls_handle_t vlsh, const void *buf, size_t n,
 	       vppcom_endpt_tlv_t *ep_tlv, int flags,
-	       __CONST_SOCKADDR_ARG addr, socklen_t addr_len)
+	       __CONST_SOCKADDR_ARG _addr, socklen_t addr_len)
 {
+  const struct sockaddr *addr = SOCKADDR_GET_SA (_addr);
   vppcom_endpt_t *ep = 0;
   vppcom_endpt_t _ep;
 
@@ -1582,11 +1597,11 @@ ldp_vls_sendo (vls_handle_t vlsh, const void *buf, size_t n,
 }
 
 static int
-ldp_vls_recvfrom (vls_handle_t vlsh, void *__restrict buf, size_t n,
-		  int flags, __SOCKADDR_ARG addr,
-		  socklen_t * __restrict addr_len)
+ldp_vls_recvfrom (vls_handle_t vlsh, void *__restrict buf, size_t n, int flags,
+		  __SOCKADDR_ARG _addr, socklen_t *__restrict addr_len)
 {
   u8 src_addr[sizeof (struct sockaddr_in6)];
+  struct sockaddr *addr = SOCKADDR_GET_SA (_addr);
   vppcom_endpt_t ep;
   ssize_t size;
   int rv;
@@ -1611,8 +1626,9 @@ ldp_vls_recvfrom (vls_handle_t vlsh, void *__restrict buf, size_t n,
 
 ssize_t
 sendto (int fd, const void *buf, size_t n, int flags,
-	__CONST_SOCKADDR_ARG addr, socklen_t addr_len)
+	__CONST_SOCKADDR_ARG _addr, socklen_t addr_len)
 {
+  const struct sockaddr *addr = SOCKADDR_GET_SA (_addr);
   vls_handle_t vlsh;
   ssize_t size;
 
@@ -1723,7 +1739,7 @@ sendmsg (int fd, const struct msghdr * msg, int flags)
   return size;
 }
 
-#ifdef USE_GNU
+#ifdef _GNU_SOURCE
 int
 sendmmsg (int fd, struct mmsghdr *vmessages, unsigned int vlen, int flags)
 {
@@ -1822,52 +1838,60 @@ recvmsg (int fd, struct msghdr * msg, int flags)
   return size;
 }
 
-#ifdef USE_GNU
+#ifdef _GNU_SOURCE
 int
 recvmmsg (int fd, struct mmsghdr *vmessages,
 	  unsigned int vlen, int flags, struct timespec *tmo)
 {
-  ssize_t size;
-  const char *func_str;
-  u32 sh = ldp_fd_to_vlsh (fd);
+  ldp_worker_ctx_t *ldpw = ldp_worker_get_current ();
+  u32 sh;
 
   ldp_init_check ();
 
+  sh = ldp_fd_to_vlsh (fd);
+
   if (sh != VLS_INVALID_HANDLE)
     {
-      clib_warning ("LDP<%d>: LDP-TBD", getpid ());
-      errno = ENOSYS;
-      size = -1;
+      struct mmsghdr *mh;
+      ssize_t rv = 0;
+      u32 nvecs = 0;
+      f64 time_out;
+
+      if (PREDICT_FALSE (ldpw->clib_time.init_cpu_time == 0))
+	clib_time_init (&ldpw->clib_time);
+      if (tmo)
+	{
+	  time_out = (f64) tmo->tv_sec + (f64) tmo->tv_nsec / (f64) 1e9;
+	  time_out += clib_time_now (&ldpw->clib_time);
+	}
+      else
+	{
+	  time_out = (f64) ~0;
+	}
+
+      while (nvecs < vlen)
+	{
+	  mh = &vmessages[nvecs];
+	  rv = recvmsg (fd, &mh->msg_hdr, flags);
+	  if (rv > 0)
+	    {
+	      mh->msg_len = rv;
+	      nvecs += 1;
+	      continue;
+	    }
+
+	  if (!time_out || clib_time_now (&ldpw->clib_time) >= time_out)
+	    break;
+
+	  usleep (1);
+	}
+
+      return nvecs > 0 ? nvecs : rv;
     }
   else
     {
-      func_str = "libc_recvmmsg";
-
-      if (LDP_DEBUG > 2)
-	clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): "
-		      "vmessages %p, vlen %u, flags 0x%x, tmo %p",
-		      getpid (), fd, fd, func_str, vmessages, vlen,
-		      flags, tmo);
-
-      size = libc_recvmmsg (fd, vmessages, vlen, flags, tmo);
+      return libc_recvmmsg (fd, vmessages, vlen, flags, tmo);
     }
-
-  if (LDP_DEBUG > 2)
-    {
-      if (size < 0)
-	{
-	  int errno_val = errno;
-	  perror (func_str);
-	  clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s() failed! "
-			"rv %d, errno = %d", getpid (), fd, fd,
-			func_str, size, errno_val);
-	  errno = errno_val;
-	}
-      else
-	clib_warning ("LDP<%d>: fd %d (0x%x): returning %d (0x%x)",
-		      getpid (), fd, fd, size, size);
-    }
-  return size;
 }
 #endif
 
@@ -2139,9 +2163,10 @@ listen (int fd, int n)
 }
 
 static inline int
-ldp_accept4 (int listen_fd, __SOCKADDR_ARG addr,
-	     socklen_t * __restrict addr_len, int flags)
+ldp_accept4 (int listen_fd, __SOCKADDR_ARG _addr,
+	     socklen_t *__restrict addr_len, int flags)
 {
+  struct sockaddr *addr = SOCKADDR_GET_SA (_addr);
   vls_handle_t listen_vlsh, accept_vlsh;
   int rv;
 
@@ -2671,7 +2696,7 @@ done:
   return rv;
 }
 
-#ifdef USE_GNU
+#ifdef _GNU_SOURCE
 int
 ppoll (struct pollfd *fds, nfds_t nfds,
        const struct timespec *timeout, const sigset_t * sigmask)

src/vcl/ldp.h

@@ -23,9 +23,9 @@
 #define LDP_DEBUG_INIT 0
 #endif
 
+#include <vcl/ldp_glibc_socket.h>
 #include <vppinfra/error.h>
 #include <vppinfra/types.h>
-#include <vcl/ldp_glibc_socket.h>
 
 #define LDP_ENV_DEBUG     "LDP_DEBUG"
 #define LDP_ENV_APP_NAME  "LDP_APP_NAME"

src/vcl/ldp_glibc_socket.h

@@ -200,15 +200,14 @@ recvfrom (int __fd, void *__restrict __buf,
 extern ssize_t
 sendmsg (int __fd, const struct msghdr *__message, int __flags);
 
-#ifdef __USE_GNU
+#ifdef _GNU_SOURCE
 /* Send a VLEN messages as described by VMESSAGES to socket FD.
    Returns the number of datagrams successfully written or -1 for errors.
 
    This function is a cancellation point and therefore not marked with
    __THROW.  */
-extern int
-sendmmsg (int __fd, struct mmsghdr *__vmessages,
-	  unsigned int __vlen, int __flags);
+extern int sendmmsg (int __fd, struct mmsghdr *__vmessages,
+		     unsigned int __vlen, int __flags);
 #endif
 
 /* Receive a message as described by MESSAGE from socket FD.
@@ -218,7 +217,7 @@ sendmmsg (int __fd, struct mmsghdr *__vmessages,
    __THROW.  */
 extern ssize_t recvmsg (int __fd, struct msghdr *__message, int __flags);
 
-#ifdef __USE_GNU
+#ifdef _GNU_SOURCE
 /* Receive up to VLEN messages as described by VMESSAGES from socket FD.
    Returns the number of messages received or -1 for errors.
 
@@ -337,7 +336,7 @@ epoll_pwait (int __epfd, struct epoll_event *__events,
    __THROW.  */
 extern int poll (struct pollfd *__fds, nfds_t __nfds, int __timeout);
 
-#ifdef __USE_GNU
+#ifdef _GNU_SOURCE
 /* Like poll, but before waiting the threads signal mask is replaced
    with that specified in the fourth parameter.  For better usability,
    the timeout value is specified using a TIMESPEC object.

src/vcl/ldp_socket_wrapper.c

@@ -55,6 +55,10 @@
    is set.
 */
 
+#ifdef HAVE_GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
 #include <signal.h>
 #include <dlfcn.h>
 
@@ -67,7 +71,6 @@
 
 #include <vcl/ldp_socket_wrapper.h>
 
-
 enum swrap_dbglvl_e
 {
   SWRAP_LOG_ERROR = 0,
@@ -156,16 +159,14 @@ PRINTF_ATTRIBUTE (3, 4);
  * SWRAP LOADING LIBC FUNCTIONS
  *********************************************************/
 
-typedef int (*__libc_accept4) (int sockfd,
-			       struct sockaddr * addr,
-			       socklen_t * addrlen, int flags);
-typedef int (*__libc_accept) (int sockfd,
-			      struct sockaddr * addr, socklen_t * addrlen);
-typedef int (*__libc_bind) (int sockfd,
-			    const struct sockaddr * addr, socklen_t addrlen);
+typedef int (*__libc_accept4) (int sockfd, __SOCKADDR_ARG addr,
+			       socklen_t *addrlen, int flags);
+typedef int (*__libc_accept) (int sockfd, __SOCKADDR_ARG addr,
+			      socklen_t *addrlen);
+typedef int (*__libc_bind) (int sockfd, __CONST_SOCKADDR_ARG addr,
+			    socklen_t addrlen);
 typedef int (*__libc_close) (int fd);
-typedef int (*__libc_connect) (int sockfd,
-			       const struct sockaddr * addr,
+typedef int (*__libc_connect) (int sockfd, __CONST_SOCKADDR_ARG addr,
 			       socklen_t addrlen);
 
 #if 0
@@ -185,16 +186,12 @@ typedef FILE *(*__libc_fopen64) (const char *name, const char *mode);
 #ifdef HAVE_EVENTFD
 typedef int (*__libc_eventfd) (int count, int flags);
 #endif
-typedef int (*__libc_getpeername) (int sockfd,
-				   struct sockaddr * addr,
-				   socklen_t * addrlen);
-typedef int (*__libc_getsockname) (int sockfd,
-				   struct sockaddr * addr,
-				   socklen_t * addrlen);
-typedef int (*__libc_getsockopt) (int sockfd,
-				  int level,
-				  int optname,
-				  void *optval, socklen_t * optlen);
+typedef int (*__libc_getpeername) (int sockfd, __SOCKADDR_ARG addr,
+				   socklen_t *addrlen);
+typedef int (*__libc_getsockname) (int sockfd, __SOCKADDR_ARG addr,
+				   socklen_t *addrlen);
+typedef int (*__libc_getsockopt) (int sockfd, int level, int optname,
+				  void *optval, socklen_t *optlen);
 typedef int (*__libc_ioctl) (int d, unsigned long int request, ...);
 typedef int (*__libc_listen) (int sockfd, int backlog);
 typedef int (*__libc_open) (const char *pathname, int flags, mode_t mode);
@@ -204,25 +201,29 @@ typedef int (*__libc_open64) (const char *pathname, int flags, mode_t mode);
 typedef int (*__libc_openat) (int dirfd, const char *path, int flags, ...);
 typedef int (*__libc_pipe) (int pipefd[2]);
 typedef int (*__libc_read) (int fd, void *buf, size_t count);
-typedef ssize_t (*__libc_readv) (int fd, const struct iovec * iov,
-				 int iovcnt);
+typedef ssize_t (*__libc_readv) (int fd, const struct iovec *iov, int iovcnt);
 typedef int (*__libc_recv) (int sockfd, void *buf, size_t len, int flags);
-typedef int (*__libc_recvfrom) (int sockfd,
-				void *buf,
-				size_t len,
-				int flags,
-				struct sockaddr * src_addr,
-				socklen_t * addrlen);
-typedef int (*__libc_recvmsg) (int sockfd, const struct msghdr * msg,
+typedef int (*__libc_recvfrom) (int sockfd, void *buf, size_t len, int flags,
+				__SOCKADDR_ARG src_addr, socklen_t *addrlen);
+typedef int (*__libc_recvmsg) (int sockfd, const struct msghdr *msg,
 			       int flags);
+#ifdef _GNU_SOURCE
+typedef int (*__libc_recvmmsg) (int fd, struct mmsghdr *vmessages,
+				unsigned int vlen, int flags,
+				struct timespec *tmo);
+#endif
 typedef int (*__libc_send) (int sockfd, const void *buf, size_t len,
 			    int flags);
 typedef ssize_t (*__libc_sendfile) (int out_fd, int in_fd, off_t * offset,
 				    size_t len);
 typedef int (*__libc_sendmsg) (int sockfd, const struct msghdr * msg,
 			       int flags);
+#ifdef _GNU_SOURCE
+typedef int (*__libc_sendmmsg) (int __fd, struct mmsghdr *__vmessages,
+				unsigned int __vlen, int __flags);
+#endif
 typedef int (*__libc_sendto) (int sockfd, const void *buf, size_t len,
-			      int flags, const struct sockaddr * dst_addr,
+			      int flags, __CONST_SOCKADDR_ARG dst_addr,
 			      socklen_t addrlen);
 typedef int (*__libc_setsockopt) (int sockfd, int level, int optname,
 				  const void *optval, socklen_t optlen);
@@ -271,7 +272,7 @@ typedef int (*__libc_epoll_pwait) (int __epfd, struct epoll_event * __events,
 typedef int (*__libc_poll) (struct pollfd * __fds, nfds_t __nfds,
 			    int __timeout);
 
-#ifdef __USE_GNU
+#ifdef _GNU_SOURCE
 typedef int (*__libc_ppoll) (struct pollfd * __fds, nfds_t __nfds,
 			     const struct timespec * __timeout,
 			     const __sigset_t * __ss);
@@ -323,9 +324,15 @@ struct swrap_libc_symbols
   SWRAP_SYMBOL_ENTRY (recv);
   SWRAP_SYMBOL_ENTRY (recvfrom);
   SWRAP_SYMBOL_ENTRY (recvmsg);
+#ifdef _GNU_SOURCE
+  SWRAP_SYMBOL_ENTRY (recvmmsg);
+#endif
   SWRAP_SYMBOL_ENTRY (send);
   SWRAP_SYMBOL_ENTRY (sendfile);
   SWRAP_SYMBOL_ENTRY (sendmsg);
+#ifdef _GNU_SOURCE
+  SWRAP_SYMBOL_ENTRY (sendmmsg);
+#endif
   SWRAP_SYMBOL_ENTRY (sendto);
   SWRAP_SYMBOL_ENTRY (setsockopt);
 #ifdef HAVE_SIGNALFD
@@ -350,7 +357,7 @@ struct swrap_libc_symbols
   SWRAP_SYMBOL_ENTRY (epoll_wait);
   SWRAP_SYMBOL_ENTRY (epoll_pwait);
   SWRAP_SYMBOL_ENTRY (poll);
-#ifdef __USE_GNU
+#ifdef _GNU_SOURCE
   SWRAP_SYMBOL_ENTRY (ppoll);
 #endif
 };
@@ -480,8 +487,7 @@ _swrap_bind_symbol (enum swrap_lib lib, const char *fn_name)
  * So we need load each function at the point it is called the first time.
  */
 int
-libc_accept4 (int sockfd,
-	      struct sockaddr *addr, socklen_t * addrlen, int flags)
+libc_accept4 (int sockfd, __SOCKADDR_ARG addr, socklen_t *addrlen, int flags)
 {
   swrap_bind_symbol_libc (accept4);
 
@@ -489,7 +495,7 @@ libc_accept4 (int sockfd,
 }
 
 int
-libc_accept (int sockfd, struct sockaddr *addr, socklen_t * addrlen)
+libc_accept (int sockfd, __SOCKADDR_ARG addr, socklen_t *addrlen)
 {
   swrap_bind_symbol_libc (accept);
 
@@ -497,7 +503,7 @@ libc_accept (int sockfd, struct sockaddr *addr, socklen_t * addrlen)
 }
 
 int
-libc_bind (int sockfd, const struct sockaddr *addr, socklen_t addrlen)
+libc_bind (int sockfd, __CONST_SOCKADDR_ARG addr, socklen_t addrlen)
 {
   swrap_bind_symbol_libc (bind);
 
@@ -513,7 +519,7 @@ libc_close (int fd)
 }
 
 int
-libc_connect (int sockfd, const struct sockaddr *addr, socklen_t addrlen)
+libc_connect (int sockfd, __CONST_SOCKADDR_ARG addr, socklen_t addrlen)
 {
   swrap_bind_symbol_libc (connect);
 
@@ -587,7 +593,7 @@ libc_vioctl (int fd, int cmd, va_list ap)
 }
 
 int
-libc_getpeername (int sockfd, struct sockaddr *addr, socklen_t * addrlen)
+libc_getpeername (int sockfd, __SOCKADDR_ARG addr, socklen_t *addrlen)
 {
   swrap_bind_symbol_libc (getpeername);
 
@@ -595,7 +601,7 @@ libc_getpeername (int sockfd, struct sockaddr *addr, socklen_t * addrlen)
 }
 
 int
-libc_getsockname (int sockfd, struct sockaddr *addr, socklen_t * addrlen)
+libc_getsockname (int sockfd, __SOCKADDR_ARG addr, socklen_t *addrlen)
 {
   swrap_bind_symbol_libc (getsockname);
 
@@ -647,10 +653,8 @@ libc_recv (int sockfd, void *buf, size_t len, int flags)
 }
 
 int
-libc_recvfrom (int sockfd,
-	       void *buf,
-	       size_t len,
-	       int flags, struct sockaddr *src_addr, socklen_t * addrlen)
+libc_recvfrom (int sockfd, void *buf, size_t len, int flags,
+	       __SOCKADDR_ARG src_addr, socklen_t *addrlen)
 {
   swrap_bind_symbol_libc (recvfrom);
 
@@ -667,6 +671,17 @@ libc_recvmsg (int sockfd, struct msghdr *msg, int flags)
   return swrap.libc.symbols._libc_recvmsg.f (sockfd, msg, flags);
 }
 
+#ifdef _GNU_SOURCE
+int
+libc_recvmmsg (int fd, struct mmsghdr *vmessages, unsigned int vlen, int flags,
+	       struct timespec *tmo)
+{
+  swrap_bind_symbol_libc (recvmmsg);
+
+  return swrap.libc.symbols._libc_recvmmsg.f (fd, vmessages, vlen, flags, tmo);
+}
+#endif
+
 int
 libc_send (int sockfd, const void *buf, size_t len, int flags)
 {
@@ -691,11 +706,19 @@ libc_sendmsg (int sockfd, const struct msghdr *msg, int flags)
   return swrap.libc.symbols._libc_sendmsg.f (sockfd, msg, flags);
 }
 
+#ifdef _GNU_SOURCE
 int
-libc_sendto (int sockfd,
-	     const void *buf,
-	     size_t len,
-	     int flags, const struct sockaddr *dst_addr, socklen_t addrlen)
+libc_sendmmsg (int fd, struct mmsghdr *vmessages, unsigned int vlen, int flags)
+{
+  swrap_bind_symbol_libc (sendmmsg);
+
+  return swrap.libc.symbols._libc_sendmmsg.f (fd, vmessages, vlen, flags);
+}
+#endif
+
+int
+libc_sendto (int sockfd, const void *buf, size_t len, int flags,
+	     __CONST_SOCKADDR_ARG dst_addr, socklen_t addrlen)
 {
   swrap_bind_symbol_libc (sendto);
 
@@ -838,7 +861,7 @@ libc_poll (struct pollfd *__fds, nfds_t __nfds, int __timeout)
   return swrap.libc.symbols._libc_poll.f (__fds, __nfds, __timeout);
 }
 
-#ifdef __USE_GNU
+#ifdef _GNU_SOURCE
 int
 libc_ppoll (struct pollfd *__fds, nfds_t __nfds,
 	    const struct timespec *__timeout, const __sigset_t * __ss)

src/vcl/ldp_socket_wrapper.h

@@ -98,16 +98,16 @@
  * has probably something todo with with the linker.
  * So we need load each function at the point it is called the first time.
  */
-int libc_accept4 (int sockfd, struct sockaddr *addr, socklen_t * addrlen,
+int libc_accept4 (int sockfd, __SOCKADDR_ARG addr, socklen_t *addrlen,
 		  int flags);
 
-int libc_accept (int sockfd, struct sockaddr *addr, socklen_t * addrlen);
+int libc_accept (int sockfd, __SOCKADDR_ARG addr, socklen_t *addrlen);
 
-int libc_bind (int sockfd, const struct sockaddr *addr, socklen_t addrlen);
+int libc_bind (int sockfd, __CONST_SOCKADDR_ARG addr, socklen_t addrlen);
 
 int libc_close (int fd);
 
-int libc_connect (int sockfd, const struct sockaddr *addr, socklen_t addrlen);
+int libc_connect (int sockfd, __CONST_SOCKADDR_ARG addr, socklen_t addrlen);
 
 #if 0
 /* TBD: dup and dup2 to be implemented later */
@@ -128,9 +128,9 @@ int libc_vfcntl64 (int fd, int cmd, va_list ap);
 
 int libc_vioctl (int fd, int cmd, va_list ap);
 
-int libc_getpeername (int sockfd, struct sockaddr *addr, socklen_t * addrlen);
+int libc_getpeername (int sockfd, __SOCKADDR_ARG addr, socklen_t *addrlen);
 
-int libc_getsockname (int sockfd, struct sockaddr *addr, socklen_t * addrlen);
+int libc_getsockname (int sockfd, __SOCKADDR_ARG addr, socklen_t *addrlen);
 
 int
 libc_getsockopt (int sockfd,
@@ -144,25 +144,29 @@ ssize_t libc_readv (int fd, const struct iovec *iov, int iovcnt);
 
 int libc_recv (int sockfd, void *buf, size_t len, int flags);
 
-int
-libc_recvfrom (int sockfd,
-	       void *buf,
-	       size_t len,
-	       int flags, struct sockaddr *src_addr, socklen_t * addrlen);
+int libc_recvfrom (int sockfd, void *buf, size_t len, int flags,
+		   __SOCKADDR_ARG src_addr, socklen_t *addrlen);
 
 int libc_recvmsg (int sockfd, struct msghdr *msg, int flags);
 
+#ifdef _GNU_SOURCE
+int libc_recvmmsg (int fd, struct mmsghdr *vmessages, unsigned int vlen,
+		   int flags, struct timespec *tmo);
+#endif
+
 int libc_send (int sockfd, const void *buf, size_t len, int flags);
 
 ssize_t libc_sendfile (int out_fd, int in_fd, off_t * offset, size_t len);
 
 int libc_sendmsg (int sockfd, const struct msghdr *msg, int flags);
 
-int
-libc_sendto (int sockfd,
-	     const void *buf,
-	     size_t len,
-	     int flags, const struct sockaddr *dst_addr, socklen_t addrlen);
+#ifdef _GNU_SOURCE
+int libc_sendmmsg (int fd, struct mmsghdr *vmessages, unsigned int vlen,
+		   int flags);
+#endif
+
+int libc_sendto (int sockfd, const void *buf, size_t len, int flags,
+		 __CONST_SOCKADDR_ARG dst_addr, socklen_t addrlen);
 
 int
 libc_setsockopt (int sockfd,
@@ -210,7 +214,7 @@ int libc_epoll_pwait (int __epfd, struct epoll_event *__events,
 
 int libc_poll (struct pollfd *__fds, nfds_t __nfds, int __timeout);
 
-#ifdef __USE_GNU
+#ifdef _GNU_SOURCE
 int libc_ppoll (struct pollfd *__fds, nfds_t __nfds,
 		const struct timespec *__timeout, const __sigset_t * __ss);
 #endif

src/vcl/vcl_private.c

@@ -435,6 +435,7 @@ vcl_segment_attach (u64 segment_handle, char *name, ssvm_segment_type_t type,
   if ((rv = fifo_segment_attach (&vcm->segment_main, a)))
     {
       clib_warning ("svm_fifo_segment_attach ('%s') failed", name);
+      clib_rwlock_writer_unlock (&vcm->segment_table_lock);
       return rv;
     }
   hash_set (vcm->segment_table, segment_handle, a->new_segment_indices[0]);

src/vcl/vppcom.c

@@ -3034,6 +3034,7 @@ vcl_epoll_wait_handle_mq_event (vcl_worker_t * wrk, session_event_t * e,
   u64 session_evt_data = ~0;
   vcl_session_t *s;
   u8 add_event = 0;
+  svm_fifo_t *txf;
 
   switch (e->event_type)
     {
@@ -3099,9 +3100,11 @@ vcl_epoll_wait_handle_mq_event (vcl_worker_t * wrk, session_event_t * e,
       if (!(EPOLLOUT & session_events))
 	break;
       /* We didn't have a fifo when the event was added */
-      svm_fifo_add_want_deq_ntf (
-	(vcl_session_is_ct (s) ? s->ct_tx_fifo : s->tx_fifo),
-	SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL);
+      txf = vcl_session_is_ct (s) ? s->ct_tx_fifo : s->tx_fifo;
+      if (txf)
+	{
+	  svm_fifo_add_want_deq_ntf (txf, SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL);
+	}
       add_event = 1;
       events[*num_ev].events = EPOLLOUT;
       session_evt_data = s->vep.ev.data.u64;

src/vlib/main.c

@@ -1615,10 +1615,8 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
 	  if (PREDICT_FALSE (vm->elog_trace_graph_dispatch))
 	    ed = ELOG_DATA (&vlib_global_main.elog_main, es);
 
-	  nm->data_from_advancing_timing_wheel =
-	    TW (tw_timer_expire_timers_vec)
-	    ((TWT (tw_timer_wheel) *) nm->timing_wheel, vlib_time_now (vm),
-	     nm->data_from_advancing_timing_wheel);
+	  TW (tw_timer_expire_timers)
+	  ((TWT (tw_timer_wheel) *) nm->timing_wheel, vlib_time_now (vm));
 
 	  ASSERT (nm->data_from_advancing_timing_wheel != 0);
 
@@ -1649,6 +1647,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
 			vlib_get_node (vm, te->process_node_index);
 		      vlib_process_t *p =
 			vec_elt (nm->processes, n->runtime_index);
+		      p->stop_timer_handle = ~0;
 		      void *data;
 		      data =
 			vlib_process_signal_event_helper (nm, n, p,
@@ -1847,6 +1846,23 @@ vl_api_get_elog_trace_api_messages (void)
   return 0;
 }
 
+static void
+process_expired_timer_cb (u32 *expired_timer_handles)
+{
+  vlib_main_t *vm = vlib_get_main ();
+  vlib_node_main_t *nm = &vm->node_main;
+  u32 *handle;
+
+  vec_foreach (handle, expired_timer_handles)
+    {
+      u32 pi = vlib_timing_wheel_data_get_index (*handle);
+      vlib_process_t *p = vec_elt (nm->processes, pi);
+
+      p->stop_timer_handle = ~0;
+    }
+  vec_append (nm->data_from_advancing_timing_wheel, expired_timer_handles);
+}
+
 /* Main function. */
 int
 vlib_main (vlib_main_t * volatile vm, unformat_input_t * input)
@@ -1952,10 +1968,10 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input)
   vec_set_len (nm->data_from_advancing_timing_wheel, 0);
 
   /* Create the process timing wheel */
-  TW (tw_timer_wheel_init) ((TWT (tw_timer_wheel) *) nm->timing_wheel,
-			    0 /* no callback */ ,
-			    10e-6 /* timer period 10us */ ,
-			    ~0 /* max expirations per call */ );
+  TW (tw_timer_wheel_init)
+  ((TWT (tw_timer_wheel) *) nm->timing_wheel,
+   process_expired_timer_cb /* callback */, 10e-6 /* timer period 10us */,
+   ~0 /* max expirations per call */);
 
   vec_validate (vm->pending_rpc_requests, 0);
   vec_set_len (vm->pending_rpc_requests, 0);

src/vlib/node_funcs.h

@@ -998,8 +998,11 @@ vlib_process_signal_event_helper (vlib_node_main_t * nm,
       p->flags = p_flags | VLIB_PROCESS_RESUME_PENDING;
       vec_add1 (nm->data_from_advancing_timing_wheel, x);
       if (delete_from_wheel)
-	TW (tw_timer_stop) ((TWT (tw_timer_wheel) *) nm->timing_wheel,
-			    p->stop_timer_handle);
+	{
+	  TW (tw_timer_stop)
+	  ((TWT (tw_timer_wheel) *) nm->timing_wheel, p->stop_timer_handle);
+	  p->stop_timer_handle = ~0;
+	}
     }
 
   return data_to_be_written_by_caller;

src/vnet/crypto/crypto.h

@@ -338,7 +338,7 @@ typedef struct
   i16 crypto_start_offset; /* first buffer offset */
   i16 integ_start_offset;
   /* adj total_length for integ, e.g.4 bytes for IPSec ESN */
-  u16 integ_length_adj;
+  i16 integ_length_adj;
   vnet_crypto_op_status_t status : 8;
   u8 flags; /**< share same VNET_CRYPTO_OP_FLAG_* values */
 } vnet_crypto_async_frame_elt_t;
@@ -628,7 +628,7 @@ static_always_inline void
 vnet_crypto_async_add_to_frame (vlib_main_t *vm, vnet_crypto_async_frame_t *f,
 				u32 key_index, u32 crypto_len,
 				i16 integ_len_adj, i16 crypto_start_offset,
-				u16 integ_start_offset, u32 buffer_index,
+				i16 integ_start_offset, u32 buffer_index,
 				u16 next_node, u8 *iv, u8 *tag, u8 *aad,
 				u8 flags)
 {

src/vnet/ipsec/esp_encrypt.c

@@ -215,6 +215,24 @@ esp_get_ip6_hdr_len (ip6_header_t * ip6, ip6_ext_header_t ** ext_hdr)
   return len;
 }
 
+/* IPsec IV generation: IVs requirements differ depending of the
+ * encryption mode: IVs must be unpredictable for AES-CBC whereas it can
+ * be predictable but should never be reused with the same key material
+ * for CTR and GCM.
+ * We use a packet counter as the IV for CTR and GCM, and to ensure the
+ * IV is unpredictable for CBC, it is then encrypted using the same key
+ * as the message. You can refer to NIST SP800-38a and NIST SP800-38d
+ * for more details. */
+static_always_inline void *
+esp_generate_iv (ipsec_sa_t *sa, void *payload, int iv_sz)
+{
+  ASSERT (iv_sz >= sizeof (u64));
+  u64 *iv = (u64 *) (payload - iv_sz);
+  clib_memset_u8 (iv, 0, iv_sz);
+  *iv = sa->iv_counter++;
+  return iv;
+}
+
 static_always_inline void
 esp_process_chained_ops (vlib_main_t * vm, vlib_node_runtime_t * node,
 			 vnet_crypto_op_t * ops, vlib_buffer_t * b[],
@@ -368,27 +386,29 @@ esp_prepare_sync_op (vlib_main_t *vm, ipsec_per_thread_data_t *ptd,
       vnet_crypto_op_t *op;
       vec_add2_aligned (crypto_ops[0], op, 1, CLIB_CACHE_LINE_BYTES);
       vnet_crypto_op_init (op, sa0->crypto_enc_op_id);
+      u8 *crypto_start = payload;
+      /* esp_add_footer_and_icv() in esp_encrypt_inline() makes sure we always
+       * have enough space for ESP header and footer which includes ICV */
+      ASSERT (payload_len > icv_sz);
+      u16 crypto_len = payload_len - icv_sz;
+
+      /* generate the IV in front of the payload */
+      void *pkt_iv = esp_generate_iv (sa0, payload, iv_sz);
 
-      op->src = op->dst = payload;
       op->key_index = sa0->crypto_key_index;
-      op->len = payload_len - icv_sz;
       op->user_data = bi;
 
       if (ipsec_sa_is_set_IS_CTR (sa0))
 	{
-	  ASSERT (sizeof (u64) == iv_sz);
 	  /* construct nonce in a scratch space in front of the IP header */
 	  esp_ctr_nonce_t *nonce =
-	    (esp_ctr_nonce_t *) (payload - sizeof (u64) - hdr_len -
-				 sizeof (*nonce));
-	  u64 *pkt_iv = (u64 *) (payload - sizeof (u64));
-
+	    (esp_ctr_nonce_t *) (pkt_iv - hdr_len - sizeof (*nonce));
 	  if (ipsec_sa_is_set_IS_AEAD (sa0))
 	    {
 	      /* constuct aad in a scratch space in front of the nonce */
 	      op->aad = (u8 *) nonce - sizeof (esp_aead_t);
 	      op->aad_len = esp_aad_fill (op->aad, esp, sa0, seq_hi);
-	      op->tag = payload + op->len;
+	      op->tag = payload + crypto_len;
 	      op->tag_len = 16;
 	    }
 	  else
@@ -397,13 +417,17 @@ esp_prepare_sync_op (vlib_main_t *vm, ipsec_per_thread_data_t *ptd,
 	    }
 
 	  nonce->salt = sa0->salt;
-	  nonce->iv = *pkt_iv = clib_host_to_net_u64 (sa0->ctr_iv_counter++);
+	  nonce->iv = *(u64 *) pkt_iv;
 	  op->iv = (u8 *) nonce;
 	}
       else
 	{
-	  op->iv = payload - iv_sz;
-	  op->flags = VNET_CRYPTO_OP_FLAG_INIT_IV;
+	  /* construct zero iv in front of the IP header */
+	  op->iv = pkt_iv - hdr_len - iv_sz;
+	  clib_memset_u8 (op->iv, 0, iv_sz);
+	  /* include iv field in crypto */
+	  crypto_start -= iv_sz;
+	  crypto_len += iv_sz;
 	}
 
       if (lb != b[0])
@@ -412,8 +436,15 @@ esp_prepare_sync_op (vlib_main_t *vm, ipsec_per_thread_data_t *ptd,
 	  op->flags |= VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS;
 	  op->chunk_index = vec_len (ptd->chunks);
 	  op->tag = vlib_buffer_get_tail (lb) - icv_sz;
-	  esp_encrypt_chain_crypto (vm, ptd, sa0, b[0], lb, icv_sz, payload,
-				    payload_len, &op->n_chunks);
+	  esp_encrypt_chain_crypto (vm, ptd, sa0, b[0], lb, icv_sz,
+				    crypto_start, crypto_len + icv_sz,
+				    &op->n_chunks);
+	}
+      else
+	{
+	  /* not chained */
+	  op->src = op->dst = crypto_start;
+	  op->len = crypto_len;
 	}
     }
 
@@ -463,26 +494,26 @@ esp_prepare_async_frame (vlib_main_t *vm, ipsec_per_thread_data_t *ptd,
   u8 *tag, *iv, *aad = 0;
   u8 flag = 0;
   u32 key_index;
-  i16 crypto_start_offset, integ_start_offset = 0;
+  i16 crypto_start_offset, integ_start_offset;
   u16 crypto_total_len, integ_total_len;
 
   post->next_index = next;
 
   /* crypto */
-  crypto_start_offset = payload - b->data;
+  crypto_start_offset = integ_start_offset = payload - b->data;
   crypto_total_len = integ_total_len = payload_len - icv_sz;
   tag = payload + crypto_total_len;
 
   key_index = sa->linked_key_index;
 
+  /* generate the IV in front of the payload */
+  void *pkt_iv = esp_generate_iv (sa, payload, iv_sz);
+
   if (ipsec_sa_is_set_IS_CTR (sa))
     {
-      ASSERT (sizeof (u64) == iv_sz);
       /* construct nonce in a scratch space in front of the IP header */
-      esp_ctr_nonce_t *nonce = (esp_ctr_nonce_t *) (payload - sizeof (u64) -
-						    hdr_len - sizeof (*nonce));
-      u64 *pkt_iv = (u64 *) (payload - sizeof (u64));
-
+      esp_ctr_nonce_t *nonce =
+	(esp_ctr_nonce_t *) (pkt_iv - hdr_len - sizeof (*nonce));
       if (ipsec_sa_is_set_IS_AEAD (sa))
 	{
 	  /* constuct aad in a scratch space in front of the nonce */
@@ -496,13 +527,17 @@ esp_prepare_async_frame (vlib_main_t *vm, ipsec_per_thread_data_t *ptd,
 	}
 
       nonce->salt = sa->salt;
-      nonce->iv = *pkt_iv = clib_host_to_net_u64 (sa->ctr_iv_counter++);
+      nonce->iv = *(u64 *) pkt_iv;
       iv = (u8 *) nonce;
     }
   else
     {
-      iv = payload - iv_sz;
-      flag |= VNET_CRYPTO_OP_FLAG_INIT_IV;
+      /* construct zero iv in front of the IP header */
+      iv = pkt_iv - hdr_len - iv_sz;
+      clib_memset_u8 (iv, 0, iv_sz);
+      /* include iv field in crypto */
+      crypto_start_offset -= iv_sz;
+      crypto_total_len += iv_sz;
     }
 
   if (lb != b)
@@ -510,13 +545,14 @@ esp_prepare_async_frame (vlib_main_t *vm, ipsec_per_thread_data_t *ptd,
       /* chain */
       flag |= VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS;
       tag = vlib_buffer_get_tail (lb) - icv_sz;
-      crypto_total_len = esp_encrypt_chain_crypto (vm, ptd, sa, b, lb, icv_sz,
-						   payload, payload_len, 0);
+      crypto_total_len = esp_encrypt_chain_crypto (
+	vm, ptd, sa, b, lb, icv_sz, b->data + crypto_start_offset,
+	crypto_total_len + icv_sz, 0);
     }
 
   if (sa->integ_op_id)
     {
-      integ_start_offset = crypto_start_offset - iv_sz - sizeof (esp_header_t);
+      integ_start_offset -= iv_sz + sizeof (esp_header_t);
       integ_total_len += iv_sz + sizeof (esp_header_t);
 
       if (b != lb)

src/vnet/ipsec/ipsec_sa.h

@@ -137,7 +137,7 @@ typedef struct
   u32 seq;
   u32 seq_hi;
   u64 replay_window;
-  u64 ctr_iv_counter;
+  u64 iv_counter;
   dpo_id_t dpo;
 
   vnet_crypto_key_index_t crypto_key_index;