vppinfra: add heap header in front of dlmalloc space

This patch adds smal header in front of dlmalloc space, and it stores
some additional information about the heap.

Immediate benefit of this patch is that we know the underlying page size
si we can display heap page statistics / real memory usage.

Type: improvement
Change-Id: Ibd6989cc2f2f64630ab08734c9552e15029c5f3f
Signed-off-by: Damjan Marion <damarion@cisco.com>

src/svm/ssvm.c

@@ -362,8 +362,7 @@ ssvm_server_init_private (ssvm_private_t * ssvm)
 {
   uword page_size, log2_page_size, rnd_size = 0;
   ssvm_shared_header_t *sh;
-  void *oldheap;
-  u8 *heap;
+  clib_mem_heap_t *heap, *oldheap;
 
   log2_page_size = clib_mem_get_log2_page_size ();
   if (log2_page_size == 0)

src/svm/ssvm.h

@@ -142,7 +142,7 @@ ssvm_unlock_non_recursive (ssvm_shared_header_t * h)
 static inline void *
 ssvm_push_heap (ssvm_shared_header_t * sh)
 {
-  u8 *oldheap;
+  clib_mem_heap_t *oldheap;
   oldheap = clib_mem_set_heap (sh->heap);
   return ((void *) oldheap);
 }
@@ -156,7 +156,7 @@ ssvm_pop_heap (void *oldheap)
 static inline void *
 ssvm_mem_alloc (ssvm_private_t * ssvm, uword size)
 {
-  u8 *oldheap;
+  clib_mem_heap_t *oldheap;
   void *rv;
 
   oldheap = clib_mem_set_heap (ssvm->sh->heap);

src/svm/svm.h

@@ -31,7 +31,7 @@
 static inline void *
 svm_mem_alloc (svm_region_t * rp, uword size)
 {
-  u8 *oldheap;
+  clib_mem_heap_t *oldheap;
   ASSERT (rp->flags & SVM_FLAGS_MHEAP);
   u8 *rv;
 
@@ -47,7 +47,7 @@ static inline void *
 svm_mem_alloc_aligned_at_offset (svm_region_t * rp,
 				 uword size, uword align, uword offset)
 {
-  u8 *oldheap;
+  clib_mem_heap_t *oldheap;
   ASSERT (rp->flags & SVM_FLAGS_MHEAP);
   u8 *rv;
 
@@ -63,7 +63,7 @@ svm_mem_alloc_aligned_at_offset (svm_region_t * rp,
 static inline void
 svm_mem_free (svm_region_t * rp, void *ptr)
 {
-  u8 *oldheap;
+  clib_mem_heap_t *oldheap;
   ASSERT (rp->flags & SVM_FLAGS_MHEAP);
 
   pthread_mutex_lock (&rp->mutex);
@@ -77,7 +77,7 @@ svm_mem_free (svm_region_t * rp, void *ptr)
 static inline void *
 svm_push_pvt_heap (svm_region_t * rp)
 {
-  u8 *oldheap;
+  clib_mem_heap_t *oldheap;
   oldheap = clib_mem_set_heap (rp->region_heap);
   return ((void *) oldheap);
 }
@@ -85,7 +85,7 @@ svm_push_pvt_heap (svm_region_t * rp)
 static inline void *
 svm_push_data_heap (svm_region_t * rp)
 {
-  u8 *oldheap;
+  clib_mem_heap_t *oldheap;
   oldheap = clib_mem_set_heap (rp->data_heap);
   return ((void *) oldheap);
 }

src/vlib/cli.c

@@ -827,13 +827,8 @@ show_memory_usage (vlib_main_t * vm,
         /* *INDENT-OFF* */
         foreach_vlib_main (
         ({
-          void *heap = mm->per_cpu_mheaps[index];
-
           vlib_cli_output (vm, "%sThread %d %s\n", index ? "\n":"", index,
                            vlib_worker_threads[index].name);
-          vlib_cli_output (vm, "  %U\n", format_page_map,
-                           pointer_to_uword (clib_mem_get_heap_base(heap)),
-                           clib_mem_get_heap_size (heap));
           vlib_cli_output (vm, "  %U\n", format_clib_mem_heap,
                            mm->per_cpu_mheaps[index],
                            verbose);
@@ -846,8 +841,6 @@ show_memory_usage (vlib_main_t * vm,
       }
     if (numa_heaps)
       {
-	void *heap;
-
 	for (i = 0; i < ARRAY_LEN (mm->per_numa_mheaps); i++)
 	  {
 	    if (mm->per_numa_mheaps[i] == 0)
@@ -858,12 +851,8 @@ show_memory_usage (vlib_main_t * vm,
 		continue;
 	      }
 	    was_enabled = clib_mem_trace_enable_disable (0);
-	    heap = mm->per_numa_mheaps[i];
 
 	    vlib_cli_output (vm, "Numa %d:", i);
-	    vlib_cli_output (vm, "  %U\n", format_page_map,
-			     pointer_to_uword (clib_mem_get_heap_base (heap)),
-			     clib_mem_get_heap_size (heap));
 	    vlib_cli_output (vm, "  %U\n", format_clib_mem_heap,
 			     mm->per_numa_mheaps[index], verbose);
 	  }

src/vlib/threads.c

@@ -685,7 +685,7 @@ start_workers (vlib_main_t * vm)
   vlib_node_runtime_t *rt;
   u32 n_vlib_mains = tm->n_vlib_mains;
   u32 worker_thread_index;
-  u8 *main_heap = clib_mem_get_per_cpu_heap ();
+  clib_mem_heap_t *main_heap = clib_mem_get_per_cpu_heap ();
 
   vec_reset_length (vlib_worker_threads);
 

src/vppinfra/bihash_all_vector.c

@@ -17,9 +17,9 @@
 
 /* Vector of all bihashes */
 void **clib_all_bihashes;
-static void **clib_all_bihash_heap;
+static clib_mem_heap_t *clib_all_bihash_heap;
 
-void *
+clib_mem_heap_t *
 clib_all_bihash_set_heap (void)
 {
   if (PREDICT_FALSE (clib_all_bihash_heap == 0))

src/vppinfra/format.h

@@ -117,7 +117,6 @@ _(format_timeval);
 _(format_time_float);
 _(format_signal);
 _(format_ucontext_pc);
-_(format_page_map);
 #endif
 
 #undef _

src/vppinfra/linux/mem.c

@@ -542,6 +542,8 @@ clib_mem_get_page_stats (void *start, clib_mem_page_sz_t log2_page_size,
     ptr[i] = start + (i << log2_page_size);
 
   clib_memset (stats, 0, sizeof (clib_mem_page_stats_t));
+  stats->total = n_pages;
+  stats->log2_page_sz = log2_page_size;
 
   if (move_pages (0, n_pages, ptr, 0, status, 0) != 0)
     {

src/vppinfra/mem.c

@@ -71,6 +71,30 @@ clib_mem_vm_map_shared (void *base, uword size, int fd, uword offset,
   return rv;
 }
 
+u8 *
+format_clib_mem_page_stats (u8 * s, va_list * va)
+{
+  clib_mem_page_stats_t *stats = va_arg (*va, clib_mem_page_stats_t *);
+  u32 indent = format_get_indent (s) + 2;
+
+  s = format (s, "page stats: page-size %U, total %lu, mapped %lu, "
+	      "not-mapped %lu", format_log2_page_size, stats->log2_page_sz,
+	      stats->total, stats->mapped, stats->not_mapped);
+
+  if (stats->unknown)
+    s = format (s, ", unknown %lu", stats->unknown);
+
+  for (int i = 0; i < CLIB_MAX_NUMAS; i++)
+    if (stats->per_numa[i])
+      s = format (s, "\n%Unuma %u: %lu pages, %U bytes",
+		  format_white_space, indent, i,
+		  stats->per_numa[i],
+		  format_memory_size,
+		  stats->per_numa[i] << stats->log2_page_sz);
+
+  return s;
+}
+
 /*
  * fd.io coding-style-patch-verification: ON
  *

src/vppinfra/mem.h

@@ -93,6 +93,38 @@ typedef struct _clib_mem_vm_map_hdr
   struct _clib_mem_vm_map_hdr *prev, *next;
 } clib_mem_vm_map_hdr_t;
 
+#define foreach_clib_mem_heap_flag \
+  _(0, LOCKED, "locked") \
+  _(1, UNMAP_ON_DESTROY, "unmap-on-destroy")
+
+typedef enum
+{
+#define _(i, v, s) CLIB_MEM_HEAP_F_##v = (1 << i),
+  foreach_clib_mem_heap_flag
+#undef _
+} clib_mem_heap_flag_t;
+
+typedef struct
+{
+  /* base address */
+  void *base;
+
+  /* dlmalloc mspace */
+  void *mspace;
+
+  /* heap size */
+  uword size;
+
+  /* page size (log2) */
+  clib_mem_page_sz_t log2_page_sz:8;
+
+  /* flags */
+  clib_mem_heap_flag_t flags:8;
+
+  /* name - _MUST_ be last */
+  char name[0];
+} clib_mem_heap_t;
+
 typedef struct
 {
   /* log2 system page size */
@@ -122,7 +154,7 @@ extern clib_mem_main_t clib_mem_main;
 /* Unspecified NUMA socket */
 #define VEC_NUMA_UNSPECIFIED (0xFF)
 
-always_inline void *
+always_inline clib_mem_heap_t *
 clib_mem_get_per_cpu_heap (void)
 {
   int cpu = os_get_thread_index ();
@@ -130,7 +162,7 @@ clib_mem_get_per_cpu_heap (void)
 }
 
 always_inline void *
-clib_mem_set_per_cpu_heap (u8 * new_heap)
+clib_mem_set_per_cpu_heap (void *new_heap)
 {
   int cpu = os_get_thread_index ();
   void *old = clib_mem_main.per_cpu_mheaps[cpu];
@@ -146,7 +178,7 @@ clib_mem_get_per_numa_heap (u32 numa_id)
 }
 
 always_inline void *
-clib_mem_set_per_numa_heap (u8 * new_heap)
+clib_mem_set_per_numa_heap (void *new_heap)
 {
   int numa = os_get_numa_index ();
   void *old = clib_mem_main.per_numa_mheaps[numa];
@@ -187,10 +219,10 @@ always_inline void *
 clib_mem_alloc_aligned_at_offset (uword size, uword align, uword align_offset,
 				  int os_out_of_memory_on_failure)
 {
-  void *heap, *p;
-  uword cpu;
   void *mspace_get_aligned (void *msp, unsigned long n_user_data_bytes,
 			    unsigned long align, unsigned long align_offset);
+  clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
+  void *p;
 
   if (align_offset > align)
     {
@@ -200,10 +232,7 @@ clib_mem_alloc_aligned_at_offset (uword size, uword align, uword align_offset,
 	align_offset = align;
     }
 
-  cpu = os_get_thread_index ();
-  heap = clib_mem_main.per_cpu_mheaps[cpu];
-
-  p = mspace_get_aligned (heap, size, align, align_offset);
+  p = mspace_get_aligned (h->mspace, size, align, align_offset);
 
   if (PREDICT_FALSE (0 == p))
     {
@@ -270,24 +299,23 @@ clib_mem_alloc_aligned_or_null (uword size, uword align)
 always_inline uword
 clib_mem_is_heap_object (void *p)
 {
-  void *heap = clib_mem_get_per_cpu_heap ();
   int mspace_is_heap_object (void *msp, void *p);
-
-  return mspace_is_heap_object (heap, p);
+  clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
+  return mspace_is_heap_object (h->mspace, p);
 }
 
 always_inline void
 clib_mem_free (void *p)
 {
-  u8 *heap = clib_mem_get_per_cpu_heap ();
-
   void mspace_put (void *msp, void *p_arg);
+  clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
+
   /* Make sure object is in the correct heap. */
   ASSERT (clib_mem_is_heap_object (p));
 
   CLIB_MEM_POISON (p, clib_mem_size_nocheck (p));
 
-  mspace_put (heap, p);
+  mspace_put (h->mspace, p);
 }
 
 always_inline void *
@@ -324,24 +352,24 @@ clib_mem_free_s (void *p)
   clib_mem_free (p);
 }
 
-always_inline void *
+always_inline clib_mem_heap_t *
 clib_mem_get_heap (void)
 {
   return clib_mem_get_per_cpu_heap ();
 }
 
-always_inline void *
-clib_mem_set_heap (void *heap)
+always_inline clib_mem_heap_t *
+clib_mem_set_heap (clib_mem_heap_t * heap)
 {
   return clib_mem_set_per_cpu_heap (heap);
 }
 
-void clib_mem_destroy_heap (void *heap);
-void *clib_mem_create_heap (void *base, uword size, int is_locked, char *fmt,
-			    ...);
+void clib_mem_destroy_heap (clib_mem_heap_t * heap);
+clib_mem_heap_t *clib_mem_create_heap (void *base, uword size, int is_locked,
+				       char *fmt, ...);
 
 void clib_mem_main_init ();
-void *clib_mem_init (void *heap, uword size);
+void *clib_mem_init (void *base, uword size);
 void *clib_mem_init_with_page_size (uword memory_size,
 				    clib_mem_page_sz_t log2_page_sz);
 void *clib_mem_init_thread_safe (void *memory, uword memory_size);
@@ -377,14 +405,16 @@ typedef struct
   uword bytes_max;
 } clib_mem_usage_t;
 
-void clib_mem_get_heap_usage (void *heap, clib_mem_usage_t * usage);
+void clib_mem_get_heap_usage (clib_mem_heap_t * heap,
+			      clib_mem_usage_t * usage);
 
-void *clib_mem_get_heap_base (void *heap);
-uword clib_mem_get_heap_size (void *heap);
-uword clib_mem_get_heap_free_space (void *heap);
+void *clib_mem_get_heap_base (clib_mem_heap_t * heap);
+uword clib_mem_get_heap_size (clib_mem_heap_t * heap);
+uword clib_mem_get_heap_free_space (clib_mem_heap_t * heap);
 
 u8 *format_clib_mem_usage (u8 * s, va_list * args);
 u8 *format_clib_mem_heap (u8 * s, va_list * va);
+u8 *format_clib_mem_page_stats (u8 * s, va_list * va);
 
 /* Allocate virtual address space. */
 always_inline void *
@@ -454,19 +484,9 @@ u64 *clib_mem_vm_get_paddr (void *mem, clib_mem_page_sz_t log2_page_size,
 void clib_mem_destroy (void);
 int clib_mem_set_numa_affinity (u8 numa_node, int force);
 int clib_mem_set_default_numa_affinity ();
-
-typedef struct
-{
-  uword size;		/**< Map size */
-  int fd;		/**< File descriptor to be mapped */
-  uword requested_va;	/**< Request fixed position mapping */
-  void *addr;		/**< Pointer to mapped memory, if successful */
-  u8 numa_node;
-} clib_mem_vm_map_t;
-
 void clib_mem_vm_randomize_va (uword * requested_va,
 			       clib_mem_page_sz_t log2_page_size);
-void mheap_trace (void *v, int enable);
+void mheap_trace (clib_mem_heap_t * v, int enable);
 uword clib_mem_trace_enable_disable (uword enable);
 void clib_mem_trace (int enable);
 
@@ -485,6 +505,8 @@ clib_mem_round_to_page_size (uword size, clib_mem_page_sz_t log2_page_size)
 
 typedef struct
 {
+  clib_mem_page_sz_t log2_page_sz;
+  uword total;
   uword mapped;
   uword not_mapped;
   uword per_numa[CLIB_MAX_NUMAS];

src/vppinfra/unix-formats.c

@@ -964,66 +964,4 @@ unformat_unix_gid (unformat_input_t * input, va_list * args)
   return 0;
 }
 
-#define MAX_NUMNODES 16
-u8 *
-format_page_map (u8 * s, va_list * args)
-{
-  uword va = va_arg (*args, uword);
-  uword size = va_arg (*args, uword);
-  uword page_size = clib_mem_get_page_size ();
-  u32 indent = format_get_indent (s);
-  uword n_pages = size / page_size;
-  uword pages_per_numa[MAX_NUMNODES] = { 0 };
-  uword pages_not_mapped = 0;
-  uword pages_unknown = 0;
-  int *status = 0;
-  void **ptr = 0;
-  int i;
-
-  s = format (s, "virtual memory start 0x%llx, size %lluk, %u pages, "
-	      "page size %uk", va, size / 1024, n_pages, page_size / 1024);
-
-  vec_validate (status, n_pages - 1);
-  vec_validate (ptr, n_pages - 1);
-
-  for (i = 0; i < n_pages; i++)
-    ptr[i] = uword_to_pointer (va + i * page_size, void *);
-
-  if (move_pages (0, n_pages, ptr, 0, status, 0) != 0)
-    {
-      s = format (s, "\n%Upage information not available (errno %u)",
-		  format_white_space, indent + 2, errno);
-      goto done;
-    }
-
-  for (i = 0; i < n_pages; i++)
-    {
-      if (status[i] >= 0 && status[i] < MAX_NUMNODES)
-	pages_per_numa[status[i]]++;
-      else if (status[i] == -EFAULT)
-	pages_not_mapped++;
-      else
-	pages_unknown++;
-    }
-
-  for (i = 0; i < MAX_NUMNODES; i++)
-    if (pages_per_numa[i])
-      s = format (s, "\n%Unuma %u: %d pages, %luk", format_white_space,
-		  indent + 2, i, pages_per_numa[i], pages_per_numa[i] *
-		  page_size / 1024);
-
-  s = format (s, "\n%Unot mapped: %u pages, %luk", format_white_space,
-	      indent + 2, pages_not_mapped, pages_not_mapped *
-	      page_size / 1024);
-
-  if (pages_unknown)
-    s = format (s, "\n%Uunknown: %u pages, %luk", format_white_space,
-		indent + 2, pages_unknown, pages_unknown * page_size / 1024);
-
-done:
-  vec_free (status);
-  vec_free (ptr);
-  return s;
-}
-
 #endif /* __KERNEL__ */