Thread safe internal buffer manager

Change-Id: I45845b952aa42a854e1c2c396b85f905de987020
Signed-off-by: Damjan Marion <damarion@cisco.com>

src/vlib/buffer.c

@@ -305,12 +305,6 @@ vlib_buffer_validate_alloc_free (vlib_main_t * vm,
   if (CLIB_DEBUG == 0)
     return;
 
-  ASSERT (vlib_get_thread_index () == 0);
-
-  /* smp disaster check */
-  if (vec_len (vlib_mains) > 1)
-    ASSERT (vm == vlib_mains[0]);
-
   is_free = expected_state == VLIB_BUFFER_KNOWN_ALLOCATED;
   b = buffers;
   for (i = 0; i < n_buffers; i++)
@@ -1050,6 +1044,7 @@ vlib_buffer_cb_init (struct vlib_main_t *vm)
   bm->cb.vlib_buffer_free_no_next_cb = &vlib_buffer_free_no_next_internal;
   bm->cb.vlib_buffer_delete_free_list_cb =
     &vlib_buffer_delete_free_list_internal;
+  clib_spinlock_init (&bm->buffer_known_hash_lockp);
 }
 
 /** @endcond */

src/vlib/buffer.h

@@ -44,6 +44,7 @@
 #include <vppinfra/cache.h>
 #include <vppinfra/serialize.h>
 #include <vppinfra/vector.h>
+#include <vppinfra/lock.h>
 #include <vlib/error.h>		/* for vlib_error_t */
 
 #include <vlib/config.h>	/* for __PRE_DATA_SIZE */
@@ -423,6 +424,7 @@ typedef struct
      If buffer index is not in hash table then this buffer
      has never been allocated. */
   uword *buffer_known_hash;
+  clib_spinlock_t buffer_known_hash_lockp;
 
   /* List of free-lists needing Blue Light Special announcements */
   vlib_buffer_free_list_t **announce_list;

src/vlib/buffer_funcs.h

@@ -219,9 +219,10 @@ always_inline vlib_buffer_known_state_t
 vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index)
 {
   vlib_buffer_main_t *bm = vm->buffer_main;
-  ASSERT (vlib_get_thread_index () == 0);
 
+  clib_spinlock_lock (&bm->buffer_known_hash_lockp);
   uword *p = hash_get (bm->buffer_known_hash, buffer_index);
+  clib_spinlock_unlock (&bm->buffer_known_hash_lockp);
   return p ? p[0] : VLIB_BUFFER_UNKNOWN;
 }
 
@@ -231,8 +232,9 @@ vlib_buffer_set_known_state (vlib_main_t * vm,
 			     vlib_buffer_known_state_t state)
 {
   vlib_buffer_main_t *bm = vm->buffer_main;
-  ASSERT (vlib_get_thread_index () == 0);
+  clib_spinlock_lock (&bm->buffer_known_hash_lockp);
   hash_set (bm->buffer_known_hash, buffer_index, state);
+  clib_spinlock_unlock (&bm->buffer_known_hash_lockp);
 }
 
 /* Validates sanity of a single buffer.
@@ -841,10 +843,19 @@ vlib_buffer_add_to_free_list (vlib_main_t * vm,
 			      u32 buffer_index, u8 do_init)
 {
   vlib_buffer_t *b;
+  u32 i;
   b = vlib_get_buffer (vm, buffer_index);
   if (PREDICT_TRUE (do_init))
     vlib_buffer_init_for_free_list (b, f);
   vec_add1_aligned (f->buffers, buffer_index, CLIB_CACHE_LINE_BYTES);
+
+  if (vec_len (f->buffers) > 3 * VLIB_FRAME_SIZE)
+    {
+      /* keep last stored buffers, as they are more likely hot in the cache */
+      for (i = 0; i < VLIB_FRAME_SIZE; i++)
+	vm->os_physmem_free (vlib_get_buffer (vm, i));
+      vec_delete (f->buffers, VLIB_FRAME_SIZE, 0);
+    }
 }
 
 always_inline void

src/vlib/unix/physmem.c

@@ -155,7 +155,8 @@ htlb_init (vlib_main_t * vm)
 
   pm->heap = mheap_alloc_with_flags (pm->mem, pm->mem_size,
 				     /* Don't want mheap mmap/munmap with IO memory. */
-				     MHEAP_FLAG_DISABLE_VM);
+				     MHEAP_FLAG_DISABLE_VM |
+				     MHEAP_FLAG_THREAD_SAFE);
 
   cur = pointer_to_uword (pm->mem);
   i = 0;