svm: improve fifo segment prealloc support

- track fifo segment free and chunk freelist memory
- improve fifo alloc. If there are enough chunks to satisfy a fifo
  allocation request but not enough free memory, allocate a multi-chunk
  fifo
- add apis to preallocate chunks and fifo headers
- more tests

Change-Id: If18dba7ab856272c9f565d36ac36365139793e0b
Signed-off-by: Florin Coras <fcoras@cisco.com>

src/plugins/unittest/svm_fifo_test.c

@@ -1908,7 +1908,6 @@ sfifo_test_fifo_segment_hello_world (int verbose)
   a->segment_size = 256 << 10;
 
   rv = fifo_segment_create (sm, a);
-
   SFIFO_TEST (!rv, "svm_fifo_segment_create returned %d", rv);
 
   fs = fifo_segment_get_segment (sm, a->new_segment_indices[0]);
@@ -2274,11 +2273,117 @@ sfifo_test_fifo_segment_mempig (int verbose)
   return 0;
 }
 
+static int
+sfifo_test_fifo_segment_prealloc (int verbose)
+{
+  fifo_segment_create_args_t _a, *a = &_a;
+  fifo_segment_main_t *sm = &segment_main;
+  u32 max_pairs, pairs_req, free_space, pair_mem;
+  svm_fifo_t *f, *old;
+  fifo_segment_t *fs;
+  int rv;
+
+  clib_memset (a, 0, sizeof (*a));
+
+  a->segment_name = "fifo-test-prealloc";
+  a->segment_size = 256 << 10;
+  a->segment_type = SSVM_SEGMENT_MEMFD;
+
+  rv = fifo_segment_create (sm, a);
+  SFIFO_TEST (!rv, "svm_fifo_segment_create returned %d", rv);
+  fs = fifo_segment_get_segment (sm, a->new_segment_indices[0]);
+
+  /*
+   * Prealloc chunks and headers
+   */
+  free_space = fifo_segment_free_bytes (fs);
+  SFIFO_TEST (free_space <= 256 << 10, "free space expected %u is %u",
+	      256 << 10, free_space);
+  rv = fifo_segment_prealloc_fifo_chunks (fs, 4096, 50);
+  SFIFO_TEST (rv == 0, "chunk prealloc should work");
+  rv = fifo_segment_num_free_chunks (fs, 4096);
+  SFIFO_TEST (rv == 50, "prealloc chunks expected %u is %u", 50, rv);
+  rv = fifo_segment_free_bytes (fs);
+  free_space -= (sizeof (svm_fifo_chunk_t) + 4096) * 50;
+  SFIFO_TEST (rv == free_space, "free space expected %u is %u", free_space,
+	      rv);
+  rv = fifo_segment_chunk_prealloc_bytes (fs);
+  SFIFO_TEST (rv == 4096 * 50, "chunk free space expected %u is %u",
+	      4096 * 50, rv);
+
+  rv = fifo_segment_prealloc_fifo_hdrs (fs, 50);
+  SFIFO_TEST (rv == 0, "fifo hdr prealloc should work");
+  rv = fifo_segment_num_free_fifos (fs);
+  SFIFO_TEST (rv == 50, "prealloc fifo hdrs expected %u is %u", 50, rv);
+  rv = fifo_segment_free_bytes (fs);
+  free_space -= sizeof (svm_fifo_t) * 50;
+  SFIFO_TEST (rv == free_space, "free space expected %u is %u", free_space,
+	      rv);
+
+  fifo_segment_update_free_bytes (fs);
+  rv = fifo_segment_free_bytes (fs);
+  SFIFO_TEST (clib_abs (rv - (int) free_space) < 512,
+	      "free space expected %u is %u", free_space, rv);
+
+  f = fifo_segment_alloc_fifo (fs, 200 << 10, FIFO_SEGMENT_RX_FIFO);
+  SFIFO_TEST (f != 0, "fifo allocated");
+  rv = fifo_segment_num_free_chunks (fs, 4096);
+  SFIFO_TEST (rv == 0, "prealloc chunks expected %u is %u", 0, rv);
+  rv = fifo_segment_chunk_prealloc_bytes (fs);
+  SFIFO_TEST (rv == 0, "chunk free space expected %u is %u", 0, rv);
+
+  /*
+   * Multiple preallocs that consume the remaining space
+   */
+  pair_mem = 2 * (4096 + sizeof (*f) + sizeof (svm_fifo_chunk_t));
+  max_pairs = pairs_req = free_space / pair_mem - 1;
+  fifo_segment_preallocate_fifo_pairs (fs, 4096, 4096, &pairs_req);
+  SFIFO_TEST (pairs_req == 0, "prealloc pairs should work");
+  rv = fifo_segment_num_free_chunks (fs, 4096);
+  SFIFO_TEST (rv == max_pairs * 2, "prealloc chunks expected %u is %u",
+	      max_pairs * 2, rv);
+
+  rv = fifo_segment_prealloc_fifo_chunks (fs, 4096, 2);
+  SFIFO_TEST (rv == 0, "chunk prealloc should work");
+  rv = fifo_segment_num_free_chunks (fs, 4096);
+  SFIFO_TEST (rv == (max_pairs + 1) * 2, "prealloc chunks expected %u is %u",
+	      (max_pairs + 1) * 2, rv);
+
+  free_space = fifo_segment_free_bytes (fs);
+  SFIFO_TEST (rv < 8192, "free bytes expected less than %u is %u", 8192, rv);
+
+  /*
+   * Test negative prealloc cases
+   */
+  pairs_req = 1;
+  fifo_segment_preallocate_fifo_pairs (fs, 4096, 4096, &pairs_req);
+  SFIFO_TEST (pairs_req == 1, "prealloc pairs should not work");
+
+  old = f;
+  f = fifo_segment_alloc_fifo (fs, 200 << 10, FIFO_SEGMENT_RX_FIFO);
+  SFIFO_TEST (f == 0, "fifo alloc should fail");
+
+  rv = fifo_segment_prealloc_fifo_chunks (fs, 4096, 50);
+  SFIFO_TEST (rv == -1, "chunk prealloc should fail");
+
+  rv = fifo_segment_prealloc_fifo_hdrs (fs, 50);
+  SFIFO_TEST (rv == -1, "fifo hdr prealloc should fail");
+
+  /*
+   * Cleanup
+   */
+  fifo_segment_free_fifo (fs, old);
+  close (fs->ssvm.fd);
+  fifo_segment_delete (sm, fs);
+  return 0;
+}
+
 static int
 sfifo_test_fifo_segment (vlib_main_t * vm, unformat_input_t * input)
 {
   int rv, verbose = 0;
 
+  fifo_segment_main_init (&segment_main, 3ULL << 30, 5);
   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
     {
       if (unformat (input, "verbose"))
@@ -2308,6 +2413,11 @@ sfifo_test_fifo_segment (vlib_main_t * vm, unformat_input_t * input)
 	  if ((rv = sfifo_test_fifo_segment_fifo_shrink (verbose)))
 	    return -1;
 	}
+      else if (unformat (input, "prealloc"))
+	{
+	  if ((rv = sfifo_test_fifo_segment_prealloc (verbose)))
+	    return -1;
+	}
       else if (unformat (input, "all"))
 	{
 	  if ((rv = sfifo_test_fifo_segment_hello_world (verbose)))
@@ -2318,6 +2428,8 @@ sfifo_test_fifo_segment (vlib_main_t * vm, unformat_input_t * input)
 	    return -1;
 	  if ((rv = sfifo_test_fifo_segment_fifo_shrink (verbose)))
 	    return -1;
+	  if ((rv = sfifo_test_fifo_segment_prealloc (verbose)))
+	    return -1;
 	  /* Pretty slow so avoid running it always
 	     if ((rv = sfifo_test_fifo_segment_master_slave (verbose)))
 	     return -1;
@@ -2340,6 +2452,7 @@ svm_fifo_test (vlib_main_t * vm, unformat_input_t * input,
   int res = 0;
   char *str;
 
+
   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
     {
       if (unformat (input, "fifo1"))

src/svm/fifo_segment.h

@@ -43,6 +43,8 @@ typedef struct
   svm_fifo_chunk_t **free_chunks;	/**< Freelists by chunk size */
   u32 n_active_fifos;			/**< Number of active fifos */
   u8 flags;				/**< Segment flags */
+  u32 n_free_bytes;			/**< Bytes usable for new allocs */
+  u32 n_fl_chunk_bytes;			/**< Chunk bytes on freelist */
 } fifo_segment_header_t;
 
 typedef struct
@@ -100,6 +102,30 @@ svm_fifo_t *fifo_segment_alloc_fifo (fifo_segment_t * fs,
  */
 void fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f);
 
+/**
+ * Try to preallocate fifo headers
+ *
+ * Tries to preallocate fifo headers and adds them to freelist.
+ *
+ * @param fs		fifo segment
+ * @param batch_size	number of chunks to be allocated
+ * @return		0 on success, negative number otherwise
+ */
+int fifo_segment_prealloc_fifo_hdrs (fifo_segment_t * fs, u32 batch_size);
+
+/**
+ * Try to preallocate fifo chunks on segment
+ *
+ * Tries to preallocate chunks of requested size on segment and adds them
+ * to chunk freelist.
+ *
+ * @param fs		fifo segment
+ * @param chunk_size	size of chunks to be allocated in bytes
+ * @param batch_size	number of chunks to be allocated
+ * @return		0 on success, negative number otherwise
+ */
+int fifo_segment_prealloc_fifo_chunks (fifo_segment_t * fs, u32 chunk_size,
+				       u32 batch_size);
 /**
  * Pre-allocates fifo pairs in fifo segment
  *
@@ -137,10 +163,40 @@ int fifo_segment_grow_fifo (fifo_segment_t * fs, svm_fifo_t * f,
  * @return		0 on success, error otherwise
  */
 int fifo_segment_collect_fifo_chunks (fifo_segment_t * fs, svm_fifo_t * f);
+
+/**
+ * Fifo segment estimate of number of free bytes
+ *
+ * Returns fifo segment's internal estimate of the number of free bytes.
+ * To force a synchronization between the segment and the underlying
+ * memory allocator, call @ref fifo_segment_update_free_bytes
+ *
+ * @param fs		fifo segment
+ * @return		free bytes estimate
+ */
+u32 fifo_segment_free_bytes (fifo_segment_t * fs);
+
+/**
+ * Update fifo segment free bytes estimate
+ *
+ * Forces fifo segment free bytes estimate synchronization with underlying
+ * memory allocator.
+ *
+ * @param fs		fifo segment
+ */
+void fifo_segment_update_free_bytes (fifo_segment_t * fs);
+
+/**
+ * Number of bytes on chunk free lists
+ *
+ * @param fs		fifo segment
+ * @return		free bytes on chunk free lists
+ */
+u32 fifo_segment_chunk_prealloc_bytes (fifo_segment_t * fs);
 u8 fifo_segment_has_fifos (fifo_segment_t * fs);
 svm_fifo_t *fifo_segment_get_fifo_list (fifo_segment_t * fs);
 u32 fifo_segment_num_fifos (fifo_segment_t * fs);
-u32 fifo_segment_num_free_fifos (fifo_segment_t * fs, u32 fifo_size_in_bytes);
+u32 fifo_segment_num_free_fifos (fifo_segment_t * fs);
 /**
  * Find number of free chunks of given size
  *

src/svm/ssvm.c

@@ -352,8 +352,8 @@ ssvm_master_init_private (ssvm_private_t * ssvm)
   u32 rnd_size = 0;
   u8 *heap;
 
-  rnd_size = (ssvm->ssvm_size + (pagesize - 1)) & ~(pagesize - 1);
-  rnd_size = clib_min (rnd_size, ((u64) 1 << 32) - pagesize);
+  rnd_size = clib_max (ssvm->ssvm_size + (pagesize - 1), ssvm->ssvm_size);
+  rnd_size &= ~(pagesize - 1);
 
 #if USE_DLMALLOC == 0
   {

src/vnet/session/segment_manager.c

@@ -102,7 +102,7 @@ segment_manager_add_segment (segment_manager_t * sm, u32 segment_size)
     }
 
   /*
-   * Allocate fifo segment and lock if needed
+   * Allocate fifo segment and grab lock if needed
    */
   if (vlib_num_workers ())
     clib_rwlock_writer_lock (&sm->segments_rwlock);
@@ -110,13 +110,14 @@ segment_manager_add_segment (segment_manager_t * sm, u32 segment_size)
   pool_get_zero (sm->segments, fs);
 
   /*
-   * Initialize ssvm segment and svm fifo private header
+   * Allocate ssvm segment
    */
   segment_size = segment_size ? segment_size : props->add_segment_size;
   page_size = clib_mem_get_page_size ();
   /* Protect against segment size u32 wrap */
   segment_size = clib_max (segment_size + page_size - 1, segment_size);
   segment_size = segment_size & ~(page_size - 1);
+
   if (props->segment_type != SSVM_SEGMENT_PRIVATE)
     {
       seg_name = format (0, "%d-%d%c", getpid (), smm->seg_name_counter++, 0);
@@ -147,6 +148,9 @@ segment_manager_add_segment (segment_manager_t * sm, u32 segment_size)
       goto done;
     }
 
+  /*
+   * Initialize fifo segment
+   */
   fifo_segment_init (fs);
 
   /*
@@ -761,6 +765,7 @@ segment_manager_alloc_queue (fifo_segment_t * segment,
 
   oldheap = ssvm_push_heap (segment->ssvm.sh);
   q = svm_msg_q_alloc (cfg);
+  fifo_segment_update_free_bytes (segment);
   ssvm_pop_heap (oldheap);
 
   if (props->use_mq_eventfd)
@@ -865,7 +870,7 @@ segment_manager_show_fn (vlib_main_t * vm, unformat_input_t * input,
 	  segment_manager_foreach_segment_w_lock (seg, sm, ({
 	    fifo_segment_info (seg, &address, &size);
 	    active_fifos = fifo_segment_num_fifos (seg);
-	    free_fifos = fifo_segment_num_free_fifos (seg, ~0 /* size */);
+	    free_fifos = fifo_segment_num_free_fifos (seg);
 	    vlib_cli_output (vm, "%-15v%15U%15llu%15u%15u%15llx",
 			     ssvm_name (&seg->ssvm), format_fifo_segment_type,
 			     seg, size >> 20ULL, active_fifos, free_fifos,