|
|
|
@ -41,70 +41,6 @@
|
|
|
|
|
#include <vppinfra/cache.h>
|
|
|
|
|
#include <vppinfra/os.h> /* for os_panic */
|
|
|
|
|
|
|
|
|
|
/* Per-CPU state. */
|
|
|
|
|
typedef struct {
|
|
|
|
|
/* Per-cpu local heap. */
|
|
|
|
|
void * heap;
|
|
|
|
|
|
|
|
|
|
u32 thread_id;
|
|
|
|
|
} clib_smp_per_cpu_main_t;
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
|
/* Number of CPUs used to model current computer. */
|
|
|
|
|
u32 n_cpus;
|
|
|
|
|
|
|
|
|
|
/* Number of cpus that are done and have exited. */
|
|
|
|
|
u32 n_cpus_exited;
|
|
|
|
|
|
|
|
|
|
/* Log2 stack and vm (heap) size. */
|
|
|
|
|
u8 log2_n_per_cpu_stack_bytes, log2_n_per_cpu_vm_bytes;
|
|
|
|
|
|
|
|
|
|
/* Thread local store (TLS) is stored at stack top.
|
|
|
|
|
Number of 4k pages to allocate for TLS. */
|
|
|
|
|
u16 n_tls_4k_pages;
|
|
|
|
|
|
|
|
|
|
/* Per cpus stacks/heaps start at these addresses. */
|
|
|
|
|
void * vm_base;
|
|
|
|
|
|
|
|
|
|
/* Thread-safe global heap. Objects here can be allocated/freed by any cpu. */
|
|
|
|
|
void * global_heap;
|
|
|
|
|
|
|
|
|
|
clib_smp_per_cpu_main_t * per_cpu_mains;
|
|
|
|
|
} clib_smp_main_t;
|
|
|
|
|
|
|
|
|
|
extern clib_smp_main_t clib_smp_main;
|
|
|
|
|
|
|
|
|
|
always_inline void *
|
|
|
|
|
clib_smp_vm_base_for_cpu (clib_smp_main_t * m, uword cpu)
|
|
|
|
|
{
|
|
|
|
|
return m->vm_base + (cpu << m->log2_n_per_cpu_vm_bytes);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
always_inline void *
|
|
|
|
|
clib_smp_stack_top_for_cpu (clib_smp_main_t * m, uword cpu)
|
|
|
|
|
{
|
|
|
|
|
/* Stack is at top of per cpu VM area. */
|
|
|
|
|
return clib_smp_vm_base_for_cpu (m, cpu + 1) - ((uword) 1 << m->log2_n_per_cpu_stack_bytes);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
always_inline uword
|
|
|
|
|
os_get_cpu_number_inline (void)
|
|
|
|
|
{
|
|
|
|
|
clib_smp_main_t * m = &clib_smp_main;
|
|
|
|
|
void * sp;
|
|
|
|
|
uword n;
|
|
|
|
|
|
|
|
|
|
/* Get any old stack address. */
|
|
|
|
|
sp = &sp;
|
|
|
|
|
|
|
|
|
|
n = ((uword)sp - (uword)m->vm_base) >> m->log2_n_per_cpu_vm_bytes;
|
|
|
|
|
|
|
|
|
|
if (CLIB_DEBUG && m->n_cpus > 0 && n >= m->n_cpus)
|
|
|
|
|
os_panic ();
|
|
|
|
|
|
|
|
|
|
return n < m->n_cpus ? n : 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#define clib_smp_compare_and_swap(addr,new,old) __sync_val_compare_and_swap(addr,old,new)
|
|
|
|
|
#define clib_smp_swap(addr,new) __sync_lock_test_and_set(addr,new)
|
|
|
|
|
#define clib_smp_atomic_add(addr,increment) __sync_fetch_and_add(addr,increment)
|
|
|
|
@ -129,217 +65,5 @@ os_sched_yield (void)
|
|
|
|
|
{ clib_smp_pause (); }
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
typedef enum {
|
|
|
|
|
CLIB_SMP_LOCK_TYPE_READER,
|
|
|
|
|
CLIB_SMP_LOCK_TYPE_WRITER,
|
|
|
|
|
CLIB_SMP_LOCK_TYPE_SPIN,
|
|
|
|
|
} clib_smp_lock_type_t;
|
|
|
|
|
|
|
|
|
|
typedef enum {
|
|
|
|
|
CLIB_SMP_LOCK_WAIT_EMPTY,
|
|
|
|
|
CLIB_SMP_LOCK_WAIT_DONE,
|
|
|
|
|
CLIB_SMP_LOCK_WAIT_READER,
|
|
|
|
|
CLIB_SMP_LOCK_WAIT_WRITER,
|
|
|
|
|
} clib_smp_lock_wait_type_t;
|
|
|
|
|
|
|
|
|
|
#if uword_bits == 64
|
|
|
|
|
typedef u16 clib_smp_quarter_word_t;
|
|
|
|
|
typedef u32 clib_smp_half_word_t;
|
|
|
|
|
#else
|
|
|
|
|
typedef u8 clib_smp_quarter_word_t;
|
|
|
|
|
typedef u16 clib_smp_half_word_t;
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
typedef union {
|
|
|
|
|
struct {
|
|
|
|
|
/* FIFO of CPUs (threads) waiting for lock. */
|
|
|
|
|
struct {
|
|
|
|
|
clib_smp_quarter_word_t head_index, n_elts;
|
|
|
|
|
} waiting_fifo;
|
|
|
|
|
|
|
|
|
|
/* Requesting CPU for atomic compare_and_swap instructions.
|
|
|
|
|
This makes CPUs requesting same header change unique. */
|
|
|
|
|
clib_smp_quarter_word_t request_cpu;
|
|
|
|
|
|
|
|
|
|
/* Count of readers who have been given read lock.
|
|
|
|
|
Not applicable for spin locks. */
|
|
|
|
|
clib_smp_quarter_word_t n_readers_with_lock : BITS (clib_smp_quarter_word_t) - 1;
|
|
|
|
|
|
|
|
|
|
/* Set when writer has been given write lock. Only one of
|
|
|
|
|
these can happen at a time. */
|
|
|
|
|
clib_smp_quarter_word_t writer_has_lock : 1;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
uword as_uword;
|
|
|
|
|
} clib_smp_lock_header_t;
|
|
|
|
|
|
|
|
|
|
always_inline uword
|
|
|
|
|
clib_smp_lock_header_is_equal (clib_smp_lock_header_t h0, clib_smp_lock_header_t h1)
|
|
|
|
|
{ return h0.as_uword == h1.as_uword; }
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
|
volatile clib_smp_lock_wait_type_t wait_type;
|
|
|
|
|
u8 pad[CLIB_CACHE_LINE_BYTES - 1 * sizeof (clib_smp_lock_wait_type_t)];
|
|
|
|
|
} clib_smp_lock_waiting_fifo_elt_t;
|
|
|
|
|
|
|
|
|
|
/* Cache aligned. */
|
|
|
|
|
typedef struct {
|
|
|
|
|
clib_smp_lock_header_t header;
|
|
|
|
|
|
|
|
|
|
/* Size of waiting FIFO; equal to max number of threads less one. */
|
|
|
|
|
u32 n_waiting_fifo_elts;
|
|
|
|
|
|
|
|
|
|
u8 pad[CLIB_CACHE_LINE_BYTES - sizeof (clib_smp_lock_header_t) - sizeof (u32)];
|
|
|
|
|
|
|
|
|
|
clib_smp_lock_waiting_fifo_elt_t waiting_fifo[0];
|
|
|
|
|
} clib_smp_lock_t;
|
|
|
|
|
|
|
|
|
|
always_inline clib_smp_lock_header_t
|
|
|
|
|
clib_smp_lock_set_header (clib_smp_lock_t * l, clib_smp_lock_header_t new_hdr, clib_smp_lock_header_t old)
|
|
|
|
|
{
|
|
|
|
|
clib_smp_lock_header_t cmp;
|
|
|
|
|
cmp.as_uword = clib_smp_compare_and_swap (&l->header.as_uword, new_hdr.as_uword, old.as_uword);
|
|
|
|
|
return cmp;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void clib_smp_lock_init (clib_smp_lock_t ** l);
|
|
|
|
|
void clib_smp_lock_free (clib_smp_lock_t ** l);
|
|
|
|
|
void clib_smp_lock_slow_path (clib_smp_lock_t * l, uword my_cpu, clib_smp_lock_header_t h, clib_smp_lock_type_t type);
|
|
|
|
|
void clib_smp_unlock_slow_path (clib_smp_lock_t * l, uword my_cpu, clib_smp_lock_header_t h, clib_smp_lock_type_t type);
|
|
|
|
|
|
|
|
|
|
always_inline void
|
|
|
|
|
clib_smp_lock_inline (clib_smp_lock_t * l, clib_smp_lock_type_t type)
|
|
|
|
|
{
|
|
|
|
|
clib_smp_lock_header_t h0, h1, h2;
|
|
|
|
|
uword is_reader = type == CLIB_SMP_LOCK_TYPE_READER;
|
|
|
|
|
uword my_cpu;
|
|
|
|
|
|
|
|
|
|
/* Null lock means n_cpus <= 1: nothing to lock. */
|
|
|
|
|
if (! l)
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
my_cpu = os_get_cpu_number_inline ();
|
|
|
|
|
h0 = l->header;
|
|
|
|
|
while (! h0.writer_has_lock)
|
|
|
|
|
{
|
|
|
|
|
/* Want to write but there are still readers with lock? */
|
|
|
|
|
if (type == CLIB_SMP_LOCK_TYPE_WRITER && h0.n_readers_with_lock != 0)
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
if (type == CLIB_SMP_LOCK_TYPE_SPIN)
|
|
|
|
|
ASSERT_AND_PANIC (h0.waiting_fifo.n_elts == 0);
|
|
|
|
|
|
|
|
|
|
/* Read/write can't proceed when waiting fifo is non-empty. */
|
|
|
|
|
else if (h0.waiting_fifo.n_elts != 0)
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
h1 = h0;
|
|
|
|
|
h1.request_cpu = my_cpu;
|
|
|
|
|
h1.writer_has_lock = ! is_reader;
|
|
|
|
|
h1.n_readers_with_lock += is_reader;
|
|
|
|
|
|
|
|
|
|
/* Try to set head and tail to zero and thereby get the lock. */
|
|
|
|
|
h2 = clib_smp_lock_set_header (l, h1, h0);
|
|
|
|
|
|
|
|
|
|
/* Compare and swap succeeded? If so, we got the lock. */
|
|
|
|
|
if (clib_smp_lock_header_is_equal (h2, h0))
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
/* Header for slow path. */
|
|
|
|
|
h0 = h2;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
clib_smp_lock_slow_path (l, my_cpu, h0, type);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
always_inline void
|
|
|
|
|
clib_smp_unlock_inline (clib_smp_lock_t * l, clib_smp_lock_type_t type)
|
|
|
|
|
{
|
|
|
|
|
clib_smp_lock_header_t h0, h1;
|
|
|
|
|
uword is_reader = type == CLIB_SMP_LOCK_TYPE_READER;
|
|
|
|
|
uword my_cpu;
|
|
|
|
|
|
|
|
|
|
/* Null means no locking is necessary. */
|
|
|
|
|
if (! l)
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
my_cpu = os_get_cpu_number_inline ();
|
|
|
|
|
h0 = l->header;
|
|
|
|
|
|
|
|
|
|
/* Should be locked. */
|
|
|
|
|
if (is_reader)
|
|
|
|
|
{
|
|
|
|
|
ASSERT_AND_PANIC (h0.n_readers_with_lock != 0);
|
|
|
|
|
ASSERT_AND_PANIC (h0.writer_has_lock == 0);
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
ASSERT_AND_PANIC (h0.n_readers_with_lock == 0);
|
|
|
|
|
ASSERT_AND_PANIC (h0.writer_has_lock);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Locked but empty waiting fifo? */
|
|
|
|
|
while (h0.waiting_fifo.n_elts == 0)
|
|
|
|
|
{
|
|
|
|
|
/* Try to mark it unlocked. */
|
|
|
|
|
h1 = h0;
|
|
|
|
|
if (is_reader)
|
|
|
|
|
h1.n_readers_with_lock -= 1;
|
|
|
|
|
else
|
|
|
|
|
h1.writer_has_lock = 0;
|
|
|
|
|
h1.request_cpu = my_cpu;
|
|
|
|
|
h1 = clib_smp_lock_set_header (l, h1, h0);
|
|
|
|
|
if (clib_smp_lock_header_is_equal (h1, h0))
|
|
|
|
|
return;
|
|
|
|
|
h0 = h1;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Other cpus are waiting. */
|
|
|
|
|
clib_smp_unlock_slow_path (l, my_cpu, h0, type);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
always_inline void
|
|
|
|
|
clib_smp_lock (clib_smp_lock_t * l)
|
|
|
|
|
{ clib_smp_lock_inline (l, CLIB_SMP_LOCK_TYPE_SPIN); }
|
|
|
|
|
|
|
|
|
|
always_inline void
|
|
|
|
|
clib_smp_lock_for_writer (clib_smp_lock_t * l)
|
|
|
|
|
{ clib_smp_lock_inline (l, CLIB_SMP_LOCK_TYPE_WRITER); }
|
|
|
|
|
|
|
|
|
|
always_inline void
|
|
|
|
|
clib_smp_lock_for_reader (clib_smp_lock_t * l)
|
|
|
|
|
{ clib_smp_lock_inline (l, CLIB_SMP_LOCK_TYPE_READER); }
|
|
|
|
|
|
|
|
|
|
always_inline void
|
|
|
|
|
clib_smp_unlock (clib_smp_lock_t * l)
|
|
|
|
|
{ clib_smp_unlock_inline (l, CLIB_SMP_LOCK_TYPE_SPIN); }
|
|
|
|
|
|
|
|
|
|
always_inline void
|
|
|
|
|
clib_smp_unlock_for_writer (clib_smp_lock_t * l)
|
|
|
|
|
{ clib_smp_unlock_inline (l, CLIB_SMP_LOCK_TYPE_WRITER); }
|
|
|
|
|
|
|
|
|
|
always_inline void
|
|
|
|
|
clib_smp_unlock_for_reader (clib_smp_lock_t * l)
|
|
|
|
|
{ clib_smp_unlock_inline (l, CLIB_SMP_LOCK_TYPE_READER); }
|
|
|
|
|
|
|
|
|
|
#define clib_exec_on_global_heap(body) \
|
|
|
|
|
do { \
|
|
|
|
|
void * __clib_exec_on_global_heap_saved_heap; \
|
|
|
|
|
\
|
|
|
|
|
/* Switch to global (thread-safe) heap. */ \
|
|
|
|
|
__clib_exec_on_global_heap_saved_heap = clib_mem_set_heap (clib_smp_main.global_heap); \
|
|
|
|
|
\
|
|
|
|
|
/* Execute body. */ \
|
|
|
|
|
body; \
|
|
|
|
|
\
|
|
|
|
|
/* Switch back to previous heap. */ \
|
|
|
|
|
clib_mem_set_heap (__clib_exec_on_global_heap_saved_heap); \
|
|
|
|
|
} while (0)
|
|
|
|
|
|
|
|
|
|
uword os_smp_bootstrap (uword n_cpus,
|
|
|
|
|
void * bootstrap_function,
|
|
|
|
|
uword bootstrap_function_arg);
|
|
|
|
|
|
|
|
|
|
void clib_smp_init (void);
|
|
|
|
|
|
|
|
|
|
#endif /* included_clib_smp_h */
|
|
|
|
|