ligang/vpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mmap-based fixed-size record double-buffered logger

Browse Source 
Change-Id: I760b482b9de457bbb17de817db7079b57d3f5ec1
Signed-off-by: Dave Barach <dave@barachs.net>
This commit is contained in:
Dave Barach
2017-11-29 16:59:01 -05:00
committed by Florin Coras
parent 4adccc9376
commit e9d9170b55
4 changed files with 364 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/vppinfra.am
+7
View File
@@ -30,6 +30,7 @@ TESTS += test_bihash_template \
test_heap \
test_longjmp \
test_macros \
test_maplog \
test_md5 \
test_mheap \
test_pool_iterate \
@@ -63,6 +64,7 @@ test_hash_SOURCES = vppinfra/test_hash.c
test_heap_SOURCES = vppinfra/test_heap.c
test_longjmp_SOURCES = vppinfra/test_longjmp.c
test_macros_SOURCES = vppinfra/test_macros.c
test_maplog_SOURCES = vppinfra/test_maplog.c
test_md5_SOURCES = vppinfra/test_md5.c
test_mheap_SOURCES = vppinfra/test_mheap.c
test_pool_iterate_SOURCES = vppinfra/test_pool_iterate.c
@@ -94,6 +96,7 @@ test_hash_CPPFLAGS = $(AM_CPPFLAGS) -DCLIB_DEBUG
test_heap_CPPFLAGS = $(AM_CPPFLAGS) -DCLIB_DEBUG
test_longjmp_CPPFLAGS = $(AM_CPPFLAGS) -DCLIB_DEBUG
test_macros_CPPFLAGS = $(AM_CPPFLAGS) -DCLIB_DEBUG
test_maplog_CPPFLAGS = $(AM_CPPFLAGS) -DCLIB_DEBUG
test_md5_CPPFLAGS = $(AM_CPPFLAGS) -DCLIB_DEBUG
test_mheap_CPPFLAGS = $(AM_CPPFLAGS) -DCLIB_DEBUG
test_pool_iterate_CPPFLAGS = $(AM_CPPFLAGS) -DCLIB_DEBUG
@@ -123,6 +126,7 @@ test_hash_LDADD = libvppinfra.la
test_heap_LDADD = libvppinfra.la
test_longjmp_LDADD = libvppinfra.la
test_macros_LDADD = libvppinfra.la
test_maplog_LDADD = libvppinfra.la
test_md5_LDADD = libvppinfra.la
test_mheap_LDADD = libvppinfra.la
test_pool_iterate_LDADD = libvppinfra.la
@@ -152,6 +156,7 @@ test_hash_LDFLAGS = -static
test_heap_LDFLAGS = -static
test_longjmp_LDFLAGS = -static
test_macros_LDFLAGS = -static
test_maplog_LDFLAGS = -static
test_md5_LDFLAGS = -static
test_mheap_LDFLAGS = -static
test_pool_iterate_LDFLAGS = -static
@@ -209,6 +214,7 @@ nobase_include_HEADERS = \
vppinfra/lock.h \
vppinfra/longjmp.h \
vppinfra/macros.h \
vppinfra/maplog.h \
vppinfra/math.h \
vppinfra/md5.h \
vppinfra/mem.h \
@@ -311,6 +317,7 @@ libvppinfra_la_SOURCES = \
vppinfra/elf_clib.c \
vppinfra/linux/mem.c \
vppinfra/linux/sysfs.c \
vppinfra/maplog.c \
vppinfra/socket.c \
vppinfra/timer.c \
vppinfra/unix-formats.c \
src/vppinfra/maplog.c
+183
View File
@@ -0,0 +1,183 @@
/*
* Copyright (c) 2017 Cisco and/or its affiliates.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <vppinfra/maplog.h>
int
clib_maplog_init (clib_maplog_main_t * mm, char *file_basename,
u64 file_size_in_bytes, u32 record_size_in_bytes)
{
int i, fd;
int rv = 0;
u8 zero = 0;
u32 record_size_in_cache_lines;
u64 file_size_in_records;
/* Already initialized? */
if (mm->flags & CLIB_MAPLOG_FLAG_INIT)
return (-2);
memset (mm, 0, sizeof (*mm));
record_size_in_cache_lines =
(record_size_in_bytes + CLIB_CACHE_LINE_BYTES -
1) / CLIB_CACHE_LINE_BYTES;
file_size_in_records = file_size_in_bytes
/ (record_size_in_cache_lines * CLIB_CACHE_LINE_BYTES);
/* Round up file size in records to a power of 2, for speed... */
mm->log2_file_size_in_records = max_log2 (file_size_in_records);
file_size_in_records = 1ULL << (mm->log2_file_size_in_records);
file_size_in_bytes = file_size_in_records * record_size_in_cache_lines
* CLIB_CACHE_LINE_BYTES;
mm->file_basename = format (0, "%s", file_basename);
mm->file_size_in_records = file_size_in_records;
mm->flags |= CLIB_MAPLOG_FLAG_INIT;
mm->record_size_in_cachelines = record_size_in_cache_lines;
/* Map two files */
for (i = 0; i < 2; i++)
{
mm->filenames[i] = format (0, "%v_%d", mm->file_basename,
mm->current_file_index++);
fd = open ((char *) mm->filenames[i], O_CREAT | O_RDWR | O_TRUNC, 0600);
if (fd < 0)
{
rv = -3;
goto fail;
}
if (lseek (fd, file_size_in_bytes - 1, SEEK_SET) == (off_t) - 1)
{
rv = -4;
goto fail;
}
if (write (fd, &zero, 1) != 1)
{
rv = -5;
goto fail;
}
mm->file_baseva[i] = mmap (0, file_size_in_bytes,
PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
if (mm->file_baseva[i] == (u8 *) MAP_FAILED)
{
clib_unix_warning ("mmap");
goto fail;
}
(void) close (fd);
}
return rv;
fail:
if (fd)
(void) close (fd);
for (i = 0; i < 2; i++)
{
if (mm->file_baseva[i])
(void) munmap ((u8 *) mm->file_baseva[i], file_size_in_bytes);
}
return rv;
}
u8 *
_clib_maplog_get_entry_slowpath (clib_maplog_main_t * mm, u64 my_record_index)
{
int fd;
u8 *rv;
u8 zero = 0;
u32 unmap_index = (mm->current_file_index) & 1;
u64 file_size_in_bytes = mm->file_size_in_records
* mm->record_size_in_cachelines * CLIB_CACHE_LINE_BYTES;
(void) munmap ((u8 *) mm->file_baseva[unmap_index], file_size_in_bytes);
vec_reset_length (mm->filenames[unmap_index]);
mm->filenames[unmap_index] = format (mm->filenames[unmap_index],
"%v_%d", mm->file_basename,
mm->current_file_index++);
fd = open ((char *) mm->filenames[unmap_index],
O_CREAT | O_RDWR | O_TRUNC, 0600);
/* $$$ this is not real error recovery... */
if (fd < 0)
{
clib_unix_warning ("creat");
abort ();
}
if (lseek (fd, file_size_in_bytes - 1, SEEK_SET) == (off_t) - 1)
{
clib_unix_warning ("lseek");
abort ();
}
if (write (fd, &zero, 1) != 1)
{
clib_unix_warning ("set-size write");
abort ();
}
mm->file_baseva[unmap_index] =
mmap (0, file_size_in_bytes, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
if (mm->file_baseva[unmap_index] == (u8 *) MAP_FAILED)
{
clib_unix_warning ("mmap");
abort ();
}
(void) close (fd);
rv = (u8 *)
mm->file_baseva[(my_record_index >> mm->log2_file_size_in_records) & 1] +
(my_record_index & (mm->file_size_in_records - 1))
* mm->record_size_in_cachelines * CLIB_CACHE_LINE_BYTES;
return rv;
}
void
clib_maplog_close (clib_maplog_main_t * mm)
{
int i;
u64 file_size_in_bytes;
if (!(mm->flags & CLIB_MAPLOG_FLAG_INIT))
return;
file_size_in_bytes =
mm->file_size_in_records * mm->record_size_in_cachelines *
CLIB_CACHE_LINE_BYTES;
for (i = 0; i < 2; i++)
{
(void) munmap ((u8 *) mm->file_baseva[i], file_size_in_bytes);
vec_free (mm->filenames[i]);
}
vec_free (mm->file_basename);
memset (mm, 0, sizeof (*mm));
}
/*
* fd.io coding-style-patch-verification: ON
*
* Local Variables:
* eval: (c-set-style "gnu")
* End:
*/
src/vppinfra/maplog.h
+99
View File
@@ -0,0 +1,99 @@
/*
* Copyright (c) 2017 Cisco and/or its affiliates.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef __included_maplog_h__
#define __included_maplog_h__
/** \file
mmap-based fixed-size record double-buffered logging
*/
#include <vppinfra/clib.h>
#include <vppinfra/cache.h>
#include <vppinfra/format.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/mman.h>
typedef struct
{
/* rw: atomic ticket-counter, file index */
CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
volatile u64 next_record_index;
u64 file_size_in_records; /**< power of two */
u32 log2_file_size_in_records;
volatile u32 current_file_index;
volatile u32 flags;
/* ro: size parameters */
CLIB_CACHE_LINE_ALIGN_MARK (cacheline2);
u32 record_size_in_cachelines;
/* double-buffered mmap'ed logfiles */
volatile u8 *file_baseva[2];
u8 *filenames[2];
/* vector not c-string */
u8 *file_basename;
} clib_maplog_main_t;
int clib_maplog_init (clib_maplog_main_t * mm, char *file_basename,
u64 file_size, u32 record_size_in_bytes);
void clib_maplog_close (clib_maplog_main_t * mm);
#define CLIB_MAPLOG_FLAG_INIT (1<<0)
u8 *_clib_maplog_get_entry_slowpath (clib_maplog_main_t * mm,
u64 my_record_index);
static inline void *
clib_maplog_get_entry (clib_maplog_main_t * mm)
{
u64 my_record_index;
u8 *rv;
ASSERT (mm->flags & CLIB_MAPLOG_FLAG_INIT);
my_record_index = __sync_fetch_and_add (&mm->next_record_index, 1);
/* Time to unmap and create a new logfile? */
if (PREDICT_FALSE ((my_record_index & (mm->file_size_in_records - 1)) == 0))
{
/* Yes, but not the very first time... (;-)... */
if (my_record_index)
return _clib_maplog_get_entry_slowpath (mm, my_record_index);
/* FALLTHROUGH */
}
rv = (u8 *)
mm->file_baseva[(my_record_index >> mm->log2_file_size_in_records) & 1] +
(my_record_index & (mm->file_size_in_records - 1))
* mm->record_size_in_cachelines * CLIB_CACHE_LINE_BYTES;
return rv;
}
#endif /* __included_maplog_h__ */
/*
* fd.io coding-style-patch-verification: ON
*
* Local Variables:
* eval: (c-set-style "gnu")
* End:
*/
src/vppinfra/test_maplog.c
+75
View File
@@ -0,0 +1,75 @@
/*
* Copyright (c) 2017 Cisco and/or its affiliates.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <vppinfra/maplog.h>
clib_maplog_main_t maplog_main;
typedef struct
{
u64 serial_number;
u64 junk[7];
} test_entry_t;
int
test_maplog_main (unformat_input_t * input)
{
clib_maplog_main_t *mm = &maplog_main;
int rv;
int i;
test_entry_t *t;
rv = clib_maplog_init (mm, "/tmp/maplog_test", 4096, sizeof (test_entry_t));
if (rv)
{
clib_warning ("clib_maplog_init returned %d", rv);
exit (1);
}
for (i = 0; i < 64 * 5; i++)
{
t = clib_maplog_get_entry (mm);
t->serial_number = i;
}
clib_maplog_close (mm);
return 0;
}
#ifdef CLIB_UNIX
int
main (int argc, char *argv[])
{
unformat_input_t i;
int ret;
unformat_init_command_line (&i, argv);
ret = test_maplog_main (&i);
unformat_free (&i);
return ret;
}
#endif /* CLIB_UNIX */
/*
* fd.io coding-style-patch-verification: ON
*
* Local Variables:
* eval: (c-set-style "gnu")
* End:
*/