ligang/vpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

VPP-236 Support 64-bit vector lengths, shared memory segments >4 GB

Browse Source 
Change-Id: I02aee33e96e7ae32094b9f82f6a667d30bb52f59
Signed-off-by: Dave Barach <dave@barachs.net>
This commit is contained in:
Dave Barach
2016-08-03 14:34:38 -04:00
committed by Keith Burns
parent c53191deb3
commit b3d93dacfd
13 changed files with 264 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
svm/svm.c
View File

@ -216,10 +216,10 @@ format_svm_region (u8 * s, va_list * args)
* rnd_pagesize
* Round to a pagesize multiple, presumably 4k works
*/
static unsigned int
rnd_pagesize (unsigned int size)
static u64
rnd_pagesize (u64 size)
{
unsigned int rv;
u64 rv;
rv = (size + (MMAP_PAGESIZE - 1)) & ~(MMAP_PAGESIZE - 1);
return (rv);
@ -235,7 +235,9 @@ svm_data_region_create (svm_map_region_args_t * a, svm_region_t * rp)
u8 junk = 0;
uword map_size;
map_size = rp->virtual_size - (MMAP_PAGESIZE + SVM_PVT_MHEAP_SIZE);
map_size = rp->virtual_size - (MMAP_PAGESIZE +
(a->pvt_heap_size ? a->pvt_heap_size :
SVM_PVT_MHEAP_SIZE));
if (a->flags & SVM_FLAGS_FILE)
{
@ -316,7 +318,9 @@ svm_data_region_map (svm_map_region_args_t * a, svm_region_t * rp)
uword map_size;
struct stat statb;
map_size = rp->virtual_size - (MMAP_PAGESIZE + SVM_PVT_MHEAP_SIZE);
map_size = rp->virtual_size -
(MMAP_PAGESIZE
+ (a->pvt_heap_size ? a->pvt_heap_size : SVM_PVT_MHEAP_SIZE));
if (a->flags & SVM_FLAGS_FILE)
{
@ -364,7 +368,9 @@ svm_data_region_map (svm_map_region_args_t * a, svm_region_t * rp)
}
ASSERT (map_size <= rp->virtual_size
- (MMAP_PAGESIZE + SVM_PVT_MHEAP_SIZE));
- (MMAP_PAGESIZE
+
(a->pvt_heap_size ? a->pvt_heap_size : SVM_PVT_MHEAP_SIZE)));
if (mmap (rp->data_base, map_size, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_FIXED, fd, 0) == MAP_FAILED)
@ -528,7 +534,9 @@ svm_map_region (svm_map_region_args_t * a)
rp->region_heap =
mheap_alloc_with_flags ((void *) (a->baseva + MMAP_PAGESIZE),
SVM_PVT_MHEAP_SIZE, MHEAP_FLAG_DISABLE_VM);
(a->pvt_heap_size != 0) ?
a->pvt_heap_size : SVM_PVT_MHEAP_SIZE,
MHEAP_FLAG_DISABLE_VM);
oldheap = svm_push_pvt_heap (rp);
rp->region_name = (char *) format (0, "%s%c", a->name, 0);
@ -542,7 +550,7 @@ svm_map_region (svm_map_region_args_t * a)
vec_validate (rp->bitmap, words - 1);
overhead_space = MMAP_PAGESIZE /* header */ +
SVM_PVT_MHEAP_SIZE;
((a->pvt_heap_size != 0)? a->pvt_heap_size : SVM_PVT_MHEAP_SIZE);
bit = 0;
data_base = (uword) rp->virtual_base;
@ -724,10 +732,9 @@ svm_mutex_cleanup (void)
}
static void
svm_region_init_internal (char *root_path, int uid, int gid)
svm_region_init_internal (svm_map_region_args_t * a)
{
svm_region_t *rp;
svm_map_region_args_t _a, *a = &_a;
u64 ticks = clib_cpu_time_now ();
uword randomize_baseva;
@ -745,14 +752,7 @@ svm_region_init_internal (char *root_path, int uid, int gid)
else
randomize_baseva = (ticks & 3) * MMAP_PAGESIZE;
memset (a, 0, sizeof (*a));
a->root_path = root_path;
a->name = SVM_GLOBAL_REGION_NAME;
a->baseva = SVM_GLOBAL_REGION_BASEVA + randomize_baseva;
a->size = SVM_GLOBAL_REGION_SIZE;
a->flags = SVM_FLAGS_NODATA;
a->uid = uid;
a->gid = gid;
a->baseva += randomize_baseva;
rp = svm_map_region (a);
ASSERT (rp);
@ -770,7 +770,7 @@ svm_region_init_internal (char *root_path, int uid, int gid)
oldheap = svm_push_pvt_heap (rp);
vec_validate (mp, 0);
mp->name_hash = hash_create_string (0, sizeof (uword));
mp->root_path = root_path ? format (0, "%s%c", root_path, 0) : 0;
mp->root_path = a->root_path ? format (0, "%s%c", a->root_path, 0) : 0;
rp->data_base = mp;
svm_pop_heap (oldheap);
}
@ -781,19 +781,58 @@ svm_region_init_internal (char *root_path, int uid, int gid)
void
svm_region_init (void)
{
svm_region_init_internal (0, 0 /* uid */ , 0 /* gid */ );
svm_map_region_args_t _a, *a = &_a;
memset (a, 0, sizeof (*a));
a->root_path = 0;
a->name = SVM_GLOBAL_REGION_NAME;
a->baseva = SVM_GLOBAL_REGION_BASEVA;
a->size = SVM_GLOBAL_REGION_SIZE;
a->flags = SVM_FLAGS_NODATA;
a->uid = 0;
a->gid = 0;
svm_region_init_internal (a);
}
void
svm_region_init_chroot (char *root_path)
{
svm_region_init_internal (root_path, 0 /* uid */ , 0 /* gid */ );
svm_map_region_args_t _a, *a = &_a;
memset (a, 0, sizeof (*a));
a->root_path = root_path;
a->name = SVM_GLOBAL_REGION_NAME;
a->baseva = SVM_GLOBAL_REGION_BASEVA;
a->size = SVM_GLOBAL_REGION_SIZE;
a->flags = SVM_FLAGS_NODATA;
a->uid = 0;
a->gid = 0;
svm_region_init_internal (a);
}
void
svm_region_init_chroot_uid_gid (char *root_path, int uid, int gid)
{
svm_region_init_internal (root_path, uid, gid);
svm_map_region_args_t _a, *a = &_a;
memset (a, 0, sizeof (*a));
a->root_path = root_path;
a->name = SVM_GLOBAL_REGION_NAME;
a->baseva = SVM_GLOBAL_REGION_BASEVA;
a->size = SVM_GLOBAL_REGION_SIZE;
a->flags = SVM_FLAGS_NODATA;
a->uid = uid;
a->gid = gid;
svm_region_init_internal (a);
}
void
svm_region_init_args (svm_map_region_args_t * a)
{
svm_region_init_internal (a);
}
void *
@ -810,7 +849,8 @@ svm_region_find_or_create (svm_map_region_args_t * a)
ASSERT (root_rp);
a->size += MMAP_PAGESIZE + SVM_PVT_MHEAP_SIZE;
a->size += MMAP_PAGESIZE +
((a->pvt_heap_size != 0) ? a->pvt_heap_size : SVM_PVT_MHEAP_SIZE);
a->size = rnd_pagesize (a->size);
region_lock (root_rp, 4);
@ -869,8 +909,8 @@ svm_region_find_or_create (svm_map_region_args_t * a)
/* Completely out of VM? */
if (index >= root_rp->bitmap_size)
{
clib_warning ("region %s: not enough VM to allocate 0x%x",
root_rp->region_name, a->size);
clib_warning ("region %s: not enough VM to allocate 0x%llx (%lld)",
root_rp->region_name, a->size, a->size);
svm_pop_heap (oldheap);
region_unlock (root_rp);
return 0;
@ -892,7 +932,7 @@ svm_region_find_or_create (svm_map_region_args_t * a)
a->baseva = root_rp->virtual_base + index * MMAP_PAGESIZE;
rp = svm_map_region (a);
pool_get (mp->subregions, subp);
name = format (0, "%s%c", a->name, 0);
subp->subregion_name = name;

6
svm/svm.h
View File

@ -71,8 +71,9 @@ typedef struct svm_map_region_args_
{
char *root_path; /* NULL means use the truly global arena */
char *name;
uword baseva;
uword size;
u64 baseva;
u64 size;
u64 pvt_heap_size;
uword flags;
char *backing_file;
uword backing_mmap_size;
@ -116,6 +117,7 @@ void *svm_region_find_or_create (svm_map_region_args_t * a);
void svm_region_init (void);
void svm_region_init_chroot (char *root_path);
void svm_region_init_chroot_uid_gid (char *root_path, int uid, int gid);
void svm_region_init_args (svm_map_region_args_t *a);
void svm_region_exit (void);
void svm_region_unmap (void *rp_arg);
void svm_client_scan (char *root_path);

9
svm/svmtool.c
View File

@ -206,10 +206,10 @@ svm_map_region_nolock (svm_map_region_args_t * a)
* rnd_pagesize
* Round to a pagesize multiple, presumably 4k works
*/
static unsigned int
rnd_pagesize (unsigned int size)
static u64
rnd_pagesize (u64 size)
{
unsigned int rv;
u64 rv;
rv = (size + (MMAP_PAGESIZE - 1)) & ~(MMAP_PAGESIZE - 1);
return (rv);
@ -247,7 +247,8 @@ svm_existing_region_map_nolock (void *root_arg, svm_map_region_args_t * a)
void *oldheap;
uword *p;
a->size += MMAP_PAGESIZE + SVM_PVT_MHEAP_SIZE;
a->size += MMAP_PAGESIZE +
(a->pvt_heap_size ? a->pvt_heap_size : SVM_PVT_MHEAP_SIZE);
a->size = rnd_pagesize (a->size);
region_lock (root_rp, 4);

18
vlib-api/vlibapi/api.h
View File

@ -141,9 +141,25 @@ typedef struct
/* vector of message ranges */
vl_api_msg_range_t *msg_ranges;
/* uid for the api shared memory region */
int api_uid;
/* gid for the api shared memory region */
int api_gid;
int api_uid;
/* base virtual address for global VM region */
u64 global_baseva;
/* size of the global VM region */
u64 global_size;
/* size of the API region */
u64 api_size;
/* size of the global VM private mheap */
u64 global_pvt_heap_size;
/* size of the api private mheap */
u64 api_pvt_heap_size;
/* Client-only data structures */
unix_shared_memory_queue_t *vl_input_queue;

5
vlib-api/vlibmemory/api.h
View File

@ -142,6 +142,11 @@ void vl_set_memory_region_name (char *name);
void vl_set_memory_root_path (char *root_path);
void vl_set_memory_uid (int uid);
void vl_set_memory_gid (int gid);
void vl_set_global_memory_baseva (u64 baseva);
void vl_set_global_memory_size (u64 size);
void vl_set_api_memory_size (u64 size);
void vl_set_global_pvt_heap_size (u64 size);
void vl_set_api_pvt_heap_size (u64 size);
void vl_enable_disable_memory_api (vlib_main_t * vm, int yesno);
void vl_client_disconnect_from_vlib (void);
int vl_client_connect_to_vlib (char *svm_name, char *client_name,

43
vlib-api/vlibmemory/memory_shared.c
View File

@ -228,6 +228,46 @@ vl_set_memory_gid (int gid)
am->api_gid = gid;
}
void
vl_set_global_memory_baseva (u64 baseva)
{
api_main_t *am = &api_main;
am->global_baseva = baseva;
}
void
vl_set_global_memory_size (u64 size)
{
api_main_t *am = &api_main;
am->global_size = size;
}
void
vl_set_api_memory_size (u64 size)
{
api_main_t *am = &api_main;
am->api_size = size;
}
void
vl_set_global_pvt_heap_size (u64 size)
{
api_main_t *am = &api_main;
am->global_pvt_heap_size = size;
}
void
vl_set_api_pvt_heap_size (u64 size)
{
api_main_t *am = &api_main;
am->api_pvt_heap_size = size;
}
int
vl_map_shmem (char *region_name, int is_vlib)
{
@ -245,10 +285,11 @@ vl_map_shmem (char *region_name, int is_vlib)
memset (a, 0, sizeof (*a));
a->name = region_name;
a->size = 16 << 20;
a->size = am->api_size ? am->api_size: (16 << 20);
a->flags = SVM_FLAGS_MHEAP;
a->uid = am->api_uid;
a->gid = am->api_gid;
a->pvt_heap_size = am->api_pvt_heap_size;
vlib_rp = svm_region_find_or_create (a);

17
vlib-api/vlibmemory/memory_vlib.c
View File

@ -1176,8 +1176,21 @@ clib_error_t *
vlibmemory_init (vlib_main_t * vm)
{
api_main_t *am = &api_main;
/* Normally NULL / 0, set by cmd line "api-segment" */
svm_region_init_chroot_uid_gid (am->root_path, am->api_uid, am->api_gid);
svm_map_region_args_t _a, *a = &_a;
memset (a, 0, sizeof (*a));
a->root_path = am->root_path;
a->name = SVM_GLOBAL_REGION_NAME;
a->baseva = (am->global_baseva != 0) ?
am->global_baseva : SVM_GLOBAL_REGION_BASEVA;
a->size = (am->global_size != 0) ? am->global_size : SVM_GLOBAL_REGION_SIZE;
a->flags = SVM_FLAGS_NODATA;
a->uid = am->api_uid;
a->gid = am->api_gid;
a->pvt_heap_size = (am->global_pvt_heap_size != 0) ? am->global_pvt_heap_size
: SVM_PVT_MHEAP_SIZE;
svm_region_init_args (a);
return 0;
}

29
vpp/vpp-api/api.c
View File

@ -8111,7 +8111,8 @@ VLIB_INIT_FUNCTION (vpe_api_init);
static clib_error_t *
api_segment_config (vlib_main_t * vm, unformat_input_t * input)
{
u8 *chroot_path;
u8 * chroot_path;
u64 baseva, size, pvt_heap_size;
int uid, gid, rv;
const int max_buf_size = 4096;
char *s, *buf;
@ -8129,7 +8130,31 @@ api_segment_config (vlib_main_t * vm, unformat_input_t * input)
else if (unformat (input, "uid %d", &uid))
vl_set_memory_uid (uid);
else if (unformat (input, "gid %d", &gid))
vl_set_memory_gid (gid);
vl_set_memory_gid (gid);
else if (unformat (input, "baseva %llx", &baseva))
vl_set_global_memory_baseva (baseva);
else if (unformat (input, "global-size %lldM", &size))
vl_set_global_memory_size (size * (1ULL<<20));
else if (unformat (input, "global-size %lldG", &size))
vl_set_global_memory_size (size * (1ULL<<30));
else if (unformat (input, "global-size %lld", &size))
vl_set_global_memory_size (size);
else if (unformat (input, "global-pvt-heap-size %lldM", &pvt_heap_size))
vl_set_global_pvt_heap_size (pvt_heap_size * (1ULL<<20));
else if (unformat (input, "global-pvt-heap-size size %lld",
&pvt_heap_size))
vl_set_global_pvt_heap_size (pvt_heap_size);
else if (unformat (input, "api-pvt-heap-size %lldM", &pvt_heap_size))
vl_set_api_pvt_heap_size (pvt_heap_size * (1ULL<<20));
else if (unformat (input, "api-pvt-heap-size size %lld",
&pvt_heap_size))
vl_set_api_pvt_heap_size (pvt_heap_size);
else if (unformat (input, "api-size %lldM", &size))
vl_set_api_memory_size (size * (1ULL<<20));
else if (unformat (input, "api-size %lldG", &size))
vl_set_api_memory_size (size * (1ULL<<30));
else if (unformat (input, "api-size %lld", &size))
vl_set_api_memory_size (size);
else if (unformat (input, "uid %s", &s))
{
/* lookup the username */

4
vpp/vpp-api/gmon.c
View File

@ -180,8 +180,8 @@ gmon_init (vlib_main_t * vm)
if ((error = vlib_call_init_function (vm, vpe_api_init)))
return (error);
/* Make sure that /global-vm is owned as directed */
svm_region_init_chroot_uid_gid (am->root_path, am->api_uid, am->api_gid);
if ((error = vlib_call_init_function(vm, vlibmemory_init)))
return(error);
gm->vlib_main = vm;

6
vppinfra/Makefile.am
View File

@ -12,7 +12,7 @@
# limitations under the License.
AUTOMAKE_OPTIONS = foreign subdir-objects
AM_CPPFLAGS = -Wall @VEC64@ # -Werror
AM_CPPFLAGS = -Wall -Werror
if WITH_UNIX
lib_LTLIBRARIES = libvppinfra.la
@ -20,7 +20,7 @@ endif
lib_LIBRARIES =
TESTS =
TESTS =
if ENABLE_TESTS
TESTS += test_bihash_template \
@ -267,5 +267,5 @@ libvppinfra_la_SOURCES = \
bin_PROGRAMS = elftool
elftool_SOURCES = tools/elftool.c
elftool_CPPFLAGS = -Wall
elftool_CPPFLAGS = $(AM_CPPFLAGS)
elftool_LDADD = libvppinfra.la -lpthread -lrt -lm

9
vppinfra/configure.ac
View File

@ -48,14 +48,5 @@ AC_ARG_WITH(unix,
AM_CONDITIONAL(WITH_UNIX, test "$with_unix" = "yes")
## Enable 64-bit vector lengths
AC_ARG_WITH(vec64,
AC_HELP_STRING([--with-vec64],[Enable 64-bit vector lengths]),
[with_vec64=1],
[with_vec64=0])
AC_SUBST(VEC64,[-DCLIB_VEC64=${with_vec64}])
AC_CONFIG_FILES([Makefile])
AC_OUTPUT

53
vppinfra/vppinfra/mheap.c
View File

@ -133,6 +133,7 @@ mheap_elt_set_size (void * v,
e = mheap_elt_at_uoffset (v, uoffset);
ASSERT (n_user_data_bytes % MHEAP_USER_DATA_WORD_BYTES == 0);
e->n_user_data = n_user_data_bytes / MHEAP_USER_DATA_WORD_BYTES;
e->is_free = is_free;
ASSERT (e->prev_n_user_data * sizeof (e->user_data[0]) >= MHEAP_MIN_USER_DATA_BYTES);
@ -152,7 +153,7 @@ always_inline void set_first_free_elt_offset (mheap_t * h, uword bin, uword uoff
i1 = (uword) 1 << (uword) (bin % BITS (h->non_empty_free_elt_heads[0]));
ASSERT (i0 < ARRAY_LEN (h->non_empty_free_elt_heads));
if (h->first_free_elt_uoffset_by_bin[bin] == ~0)
if (h->first_free_elt_uoffset_by_bin[bin] == MHEAP_GROUNDED)
h->non_empty_free_elt_heads[i0] &= ~i1;
else
h->non_empty_free_elt_heads[i0] |= i1;
@ -169,11 +170,11 @@ set_free_elt (void * v, uword uoffset, uword n_user_data_bytes)
ASSERT (n->prev_is_free);
ASSERT (e->is_free);
e->free_elt.prev_uoffset = ~0;
e->free_elt.prev_uoffset = MHEAP_GROUNDED;
e->free_elt.next_uoffset = h->first_free_elt_uoffset_by_bin[bin];
/* Fill in next free elt's previous pointer. */
if (e->free_elt.next_uoffset != ~0)
if (e->free_elt.next_uoffset != MHEAP_GROUNDED)
{
mheap_elt_t * nf = mheap_elt_at_uoffset (v, e->free_elt.next_uoffset);
ASSERT (nf->is_free);
@ -195,12 +196,17 @@ remove_free_elt (void * v, mheap_elt_t * e, uword bin)
{
mheap_t * h = mheap_header (v);
mheap_elt_t * p, * n;
#if CLIB_VEC64 > 0
u64 no, po;
#else
u32 no, po;
#endif
no = e->free_elt.next_uoffset;
n = no != ~0 ? mheap_elt_at_uoffset (v, no) : 0;
n = no != MHEAP_GROUNDED ? mheap_elt_at_uoffset (v, no) : 0;
po = e->free_elt.prev_uoffset;
p = po != ~0 ? mheap_elt_at_uoffset (v, po) : 0;
p = po != MHEAP_GROUNDED ? mheap_elt_at_uoffset (v, po) : 0;
if (! p)
set_first_free_elt_offset (h, bin, no);
@ -309,13 +315,13 @@ mheap_get_small_object (mheap_t * h, uword bin)
{
mheap_small_object_cache_t * c = &h->small_object_cache;
uword mask = mheap_small_object_cache_mask (c, bin + 1);
uword offset = ~0;
uword offset = MHEAP_GROUNDED;
if (mask)
{
uword i = min_log2 (mask);
uword o = c->offsets[i];
ASSERT (o != ~0);
ASSERT (o != MHEAP_GROUNDED);
c->bins.as_u8[i] = 0;
offset = o;
}
@ -369,7 +375,7 @@ mheap_get_search_free_bin (void * v,
word o0, o1, f0, f1, search_n_user_data_bytes;
word lo_free_usize, hi_free_usize;
ASSERT (h->first_free_elt_uoffset_by_bin[bin] != ~0);
ASSERT (h->first_free_elt_uoffset_by_bin[bin] != MHEAP_GROUNDED);
e = mheap_elt_at_uoffset (v, h->first_free_elt_uoffset_by_bin[bin]);
search_n_user_data_bytes = *n_user_data_bytes_arg;
@ -420,8 +426,8 @@ mheap_get_search_free_bin (void * v,
next:
/* Reached end of free list without finding large enough object. */
if (e->free_elt.next_uoffset == ~0)
return ~0;
if (e->free_elt.next_uoffset == MHEAP_GROUNDED)
return MHEAP_GROUNDED;
/* Otherwise keep searching for large enough object. */
e = mheap_elt_at_uoffset (v, e->free_elt.next_uoffset);
@ -517,7 +523,7 @@ mheap_get_search_free_list (void * v,
{
uword r = mheap_get_small_object (h, bin);
h->stats.n_small_object_cache_attempts += 1;
if (r != ~0)
if (r != MHEAP_GROUNDED)
{
h->stats.n_small_object_cache_hits += 1;
return r;
@ -535,12 +541,12 @@ mheap_get_search_free_list (void * v,
/* Search each occupied free bin which is large enough. */
foreach_set_bit (bi, non_empty_bin_mask, ({
uword r = mheap_get_search_free_bin (v, bi + i * BITS (uword), n_user_bytes_arg, align, align_offset);
if (r != ~0)
if (r != MHEAP_GROUNDED)
return r;
}));
}
return ~0;
return MHEAP_GROUNDED;
}
static never_inline void *
@ -586,7 +592,7 @@ mheap_get_extend_vector (void * v,
/* Make sure we have space for object plus overhead. */
if (f1 > h->max_size)
{
*offset_return = ~0;
*offset_return = MHEAP_GROUNDED;
return v;
}
@ -639,7 +645,7 @@ void * mheap_get_aligned (void * v,
/* Align offset must be multiple of minimum object size. */
if (align_offset % STRUCT_SIZE_OF (mheap_elt_t, user_data[0]) != 0)
{
*offset_return = ~0;
*offset_return = MHEAP_GROUNDED;
return v;
}
@ -663,15 +669,15 @@ void * mheap_get_aligned (void * v,
h = mheap_header (v);
/* If that fails allocate object at end of heap by extending vector. */
if (offset == ~0 && _vec_len (v) < h->max_size)
if (offset == MHEAP_GROUNDED && _vec_len (v) < h->max_size)
{
v = mheap_get_extend_vector (v, n_user_data_bytes, align, align_offset, &offset);
h = mheap_header (v);
h->stats.n_vector_expands += offset != ~0;
h->stats.n_vector_expands += offset != MHEAP_GROUNDED;
}
*offset_return = offset;
if (offset != ~0)
if (offset != MHEAP_GROUNDED)
{
h->n_elts += 1;
@ -909,7 +915,8 @@ void * mheap_alloc_with_flags (void * memory, uword memory_size, uword flags)
(clib_address_t) v, h->max_size);
/* Initialize free list heads to empty. */
memset (h->first_free_elt_uoffset_by_bin, ~0, sizeof (h->first_free_elt_uoffset_by_bin));
memset (h->first_free_elt_uoffset_by_bin, 0xFF,
sizeof (h->first_free_elt_uoffset_by_bin));
return v;
}
@ -1310,10 +1317,10 @@ void mheap_validate (void * v)
mheap_elt_t * e, * n;
uword is_first;
CHECK ((h->first_free_elt_uoffset_by_bin[i] != ~0)
CHECK ((h->first_free_elt_uoffset_by_bin[i] != MHEAP_GROUNDED)
== ((h->non_empty_free_elt_heads[i / BITS (uword)] & ((uword) 1 << (uword) (i % BITS (uword)))) != 0));
if (h->first_free_elt_uoffset_by_bin[i] == ~0)
if (h->first_free_elt_uoffset_by_bin[i] == MHEAP_GROUNDED)
continue;
e = mheap_elt_at_uoffset (v, h->first_free_elt_uoffset_by_bin[i]);
@ -1331,7 +1338,7 @@ void mheap_validate (void * v)
CHECK (n->prev_is_free);
if (is_first)
CHECK (e->free_elt.prev_uoffset == ~0);
CHECK (e->free_elt.prev_uoffset == MHEAP_GROUNDED);
is_first = 0;
s = mheap_elt_data_bytes (e);
@ -1340,7 +1347,7 @@ void mheap_validate (void * v)
free_count_from_free_lists += 1;
free_size_from_free_lists += s;
if (e->free_elt.next_uoffset == ~0)
if (e->free_elt.next_uoffset == MHEAP_GROUNDED)
break;
n = mheap_elt_at_uoffset (v, e->free_elt.next_uoffset);

49
vppinfra/vppinfra/mheap_bootstrap.h
View File

@ -50,10 +50,29 @@
typedef struct {
/* Number of mheap_size_t words of user data in previous object.
Used to find mheap_elt_t for previous object. */
#if CLIB_VEC64 > 0
u64 prev_n_user_data : 63;
/* Used to mark end/start of of doubly-linked list of mheap_elt_t's. */
#define MHEAP_N_USER_DATA_INVALID (0x7fffffffffffffffULL)
#define MHEAP_GROUNDED (~0ULL)
/* Set if previous object is free. */
u64 prev_is_free : 1;
/* Number of mheap_size_t words of user data that follow this object. */
u64 n_user_data : 63;
/* Set if this object is on free list (and therefore following free_elt
is valid). */
u64 is_free : 1;
#else
u32 prev_n_user_data : 31;
/* Used to mark end/start of of doubly-linked list of mheap_elt_t's. */
#define MHEAP_N_USER_DATA_INVALID (0x7fffffff)
#define MHEAP_GROUNDED (~0)
/* Set if previous object is free. */
u32 prev_is_free : 1;
@ -64,8 +83,22 @@ typedef struct {
/* Set if this object is on free list (and therefore following free_elt
is valid). */
u32 is_free : 1;
#endif
union {
#if CLIB_VEC64 > 0
/* For allocated objects: user data follows.
User data is allocated in units of typeof (user_data[0]). */
u64 user_data[0];
/* For free objects, offsets of next and previous free objects of this size;
~0 means end of doubly-linked list.
This is stored in user data (guaranteed to be at least 8 bytes)
but only for *free* objects. */
struct {
u64 next_uoffset, prev_uoffset;
} free_elt;
#else
/* For allocated objects: user data follows.
User data is allocated in units of typeof (user_data[0]). */
u32 user_data[0];
@ -77,6 +110,7 @@ typedef struct {
struct {
u32 next_uoffset, prev_uoffset;
} free_elt;
#endif
};
} mheap_elt_t;
@ -94,7 +128,11 @@ typedef struct {
uword callers[12];
/* Count of allocations with this traceback. */
#if CLIB_VEC64 > 0
u64 n_allocations;
#else
u32 n_allocations;
#endif
/* Count of bytes allocated with this traceback. */
u32 n_bytes;
@ -149,6 +187,11 @@ typedef struct {
#define MHEAP_HAVE_SMALL_OBJECT_CACHE 0
#endif
#if CLIB_VEC64 > 0
#undef MHEAP_HAVE_SMALL_OBJECT_CACHE
#define MHEAP_HAVE_SMALL_OBJECT_CACHE 0
#endif
/* For objects with align == 4 and align_offset == 0 (e.g. vector strings). */
typedef struct {
union {
@ -168,7 +211,11 @@ typedef struct {
/* Vec header for heaps. */
typedef struct {
/* User offsets for head of doubly-linked list of free objects of this size. */
#if CLIB_VEC64 > 0
u64 first_free_elt_uoffset_by_bin[MHEAP_N_BINS];
#else
u32 first_free_elt_uoffset_by_bin[MHEAP_N_BINS];
#endif
/* Bitmap of non-empty free list bins. */
uword non_empty_free_elt_heads[(MHEAP_N_BINS + BITS (uword) - 1) / BITS (uword)];