ligang/vpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add VLIB_NODE_FN() macro to simplify multiversioning of node functions

Browse Source 
Change-Id: Ibab5e27277f618ceb2d543b9d6a1a5f191e7d1db
Signed-off-by: Damjan Marion <damarion@cisco.com>
This commit is contained in:
Damjan Marion
2018-05-28 21:26:47 +02:00
committed by Florin Coras
parent 04e0bb2ff0
commit 812b32dd8f
19 changed files with 160 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/plugins/acl.am
+5 -5
View File
@@ -34,7 +34,7 @@ acl_plugin_la_SOURCES = \
API_FILES += acl/acl.api
if CPU_X86_64
acl_multiversioning_files = \
acl_multiversioning_sources = \
acl/dataplane_node.c
@@ -42,10 +42,10 @@ if CC_SUPPORTS_AVX2
###############################################################
# AVX2
###############################################################
libacl_plugin_avx2_la_SOURCES = $(acl_multiversioning_files)
libacl_plugin_avx2_la_SOURCES = $(acl_multiversioning_sources)
libacl_plugin_avx2_la_CFLAGS = \
$(AM_CFLAGS) @CPU_AVX2_FLAGS@ \
-DCLIB_MULTIARCH_VARIANT=avx2
-DCLIB_MARCH_VARIANT=avx2
noinst_LTLIBRARIES += libacl_plugin_avx2.la
acl_plugin_la_LIBADD += libacl_plugin_avx2.la
endif
@@ -54,10 +54,10 @@ if CC_SUPPORTS_AVX512
###############################################################
# AVX512
###############################################################
libacl_plugin_avx512_la_SOURCES = $(acl_multiversioning_files)
libacl_plugin_avx512_la_SOURCES = $(acl_multiversioning_sources)
libacl_plugin_avx512_la_CFLAGS = \
$(AM_CFLAGS) @CPU_AVX512_FLAGS@ \
-DCLIB_MULTIARCH_VARIANT=avx512
-DCLIB_MARCH_VARIANT=avx512
noinst_LTLIBRARIES += libacl_plugin_avx512.la
acl_plugin_la_LIBADD += libacl_plugin_avx512.la
endif
src/plugins/acl/dataplane_node.c
+25 -117
View File
@@ -327,36 +327,10 @@ acl_fa_node_fn (vlib_main_t * vm,
return frame->n_vectors;
}
vlib_node_function_t __clib_weak acl_in_ip4_l2_node_fn_avx512;
vlib_node_function_t __clib_weak acl_in_ip4_l2_node_fn_avx2;
vlib_node_function_t __clib_weak acl_out_ip4_l2_node_fn_avx512;
vlib_node_function_t __clib_weak acl_out_ip4_l2_node_fn_avx2;
vlib_node_function_t __clib_weak acl_in_ip6_l2_node_fn_avx512;
vlib_node_function_t __clib_weak acl_in_ip6_l2_node_fn_avx2;
vlib_node_function_t __clib_weak acl_out_ip6_l2_node_fn_avx512;
vlib_node_function_t __clib_weak acl_out_ip6_l2_node_fn_avx2;
vlib_node_function_t __clib_weak acl_in_ip4_fa_node_fn_avx512;
vlib_node_function_t __clib_weak acl_in_ip4_fa_node_fn_avx2;
vlib_node_function_t __clib_weak acl_out_ip4_fa_node_fn_avx512;
vlib_node_function_t __clib_weak acl_out_ip4_fa_node_fn_avx2;
vlib_node_function_t __clib_weak acl_in_ip6_fa_node_fn_avx512;
vlib_node_function_t __clib_weak acl_in_ip6_fa_node_fn_avx2;
vlib_node_function_t __clib_weak acl_out_ip6_fa_node_fn_avx512;
vlib_node_function_t __clib_weak acl_out_ip6_fa_node_fn_avx2;
vlib_node_registration_t acl_in_l2_ip6_node;
uword CLIB_CPU_OPTIMIZED
CLIB_MULTIARCH_FN (acl_in_ip6_l2_node_fn) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
VLIB_NODE_FN (acl_in_l2_ip6_node) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
{
acl_main_t *am = &acl_main;
return acl_fa_node_fn (vm, node, frame, 1, 1, 1,
@@ -365,10 +339,9 @@ CLIB_MULTIARCH_FN (acl_in_ip6_l2_node_fn) (vlib_main_t * vm,
}
vlib_node_registration_t acl_in_l2_ip4_node;
uword CLIB_CPU_OPTIMIZED
CLIB_MULTIARCH_FN (acl_in_ip4_l2_node_fn) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
VLIB_NODE_FN (acl_in_l2_ip4_node) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
{
acl_main_t *am = &acl_main;
return acl_fa_node_fn (vm, node, frame, 0, 1, 1,
@@ -377,10 +350,9 @@ CLIB_MULTIARCH_FN (acl_in_ip4_l2_node_fn) (vlib_main_t * vm,
}
vlib_node_registration_t acl_out_l2_ip6_node;
uword CLIB_CPU_OPTIMIZED
CLIB_MULTIARCH_FN (acl_out_ip6_l2_node_fn) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
VLIB_NODE_FN (acl_out_l2_ip6_node) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
{
acl_main_t *am = &acl_main;
return acl_fa_node_fn (vm, node, frame, 1, 0, 1,
@@ -389,10 +361,9 @@ CLIB_MULTIARCH_FN (acl_out_ip6_l2_node_fn) (vlib_main_t * vm,
}
vlib_node_registration_t acl_out_l2_ip4_node;
uword CLIB_CPU_OPTIMIZED
CLIB_MULTIARCH_FN (acl_out_ip4_l2_node_fn) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
VLIB_NODE_FN (acl_out_l2_ip4_node) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
{
acl_main_t *am = &acl_main;
return acl_fa_node_fn (vm, node, frame, 0, 0, 1,
@@ -403,93 +374,38 @@ CLIB_MULTIARCH_FN (acl_out_ip4_l2_node_fn) (vlib_main_t * vm,
/**** L3 processing path nodes ****/
vlib_node_registration_t acl_in_fa_ip6_node;
uword CLIB_CPU_OPTIMIZED
CLIB_MULTIARCH_FN (acl_in_ip6_fa_node_fn) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
VLIB_NODE_FN (acl_in_fa_ip6_node) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
{
return acl_fa_node_fn (vm, node, frame, 1, 1, 0, 0, &acl_in_fa_ip6_node);
}
vlib_node_registration_t acl_in_fa_ip4_node;
uword CLIB_CPU_OPTIMIZED
CLIB_MULTIARCH_FN (acl_in_ip4_fa_node_fn) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
VLIB_NODE_FN (acl_in_fa_ip4_node) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
{
return acl_fa_node_fn (vm, node, frame, 0, 1, 0, 0, &acl_in_fa_ip4_node);
}
vlib_node_registration_t acl_out_fa_ip6_node;
uword CLIB_CPU_OPTIMIZED
CLIB_MULTIARCH_FN (acl_out_ip6_fa_node_fn) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
VLIB_NODE_FN (acl_out_fa_ip6_node) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
{
return acl_fa_node_fn (vm, node, frame, 1, 0, 0, 0, &acl_out_fa_ip6_node);
}
vlib_node_registration_t acl_out_fa_ip4_node;
uword CLIB_CPU_OPTIMIZED
CLIB_MULTIARCH_FN (acl_out_ip4_fa_node_fn) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
VLIB_NODE_FN (acl_out_fa_ip4_node) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
{
return acl_fa_node_fn (vm, node, frame, 0, 0, 0, 0, &acl_out_fa_ip4_node);
}
#if __x86_64__
static void __clib_constructor
acl_plugin_multiarch_select (void)
{
if (acl_in_ip4_l2_node_fn_avx512 && clib_cpu_supports_avx512f ())
acl_in_l2_ip4_node.function = acl_in_ip4_l2_node_fn_avx512;
else if (acl_in_ip4_l2_node_fn_avx2 && clib_cpu_supports_avx2 ())
acl_in_l2_ip4_node.function = acl_in_ip4_l2_node_fn_avx2;
if (acl_out_ip4_l2_node_fn_avx512 && clib_cpu_supports_avx512f ())
acl_out_l2_ip4_node.function = acl_out_ip4_l2_node_fn_avx512;
else if (acl_out_ip4_l2_node_fn_avx2 && clib_cpu_supports_avx2 ())
acl_out_l2_ip4_node.function = acl_out_ip4_l2_node_fn_avx2;
if (acl_in_ip6_l2_node_fn_avx512 && clib_cpu_supports_avx512f ())
acl_in_l2_ip6_node.function = acl_in_ip6_l2_node_fn_avx512;
else if (acl_in_ip6_l2_node_fn_avx2 && clib_cpu_supports_avx2 ())
acl_in_l2_ip6_node.function = acl_in_ip6_l2_node_fn_avx2;
if (acl_out_ip6_l2_node_fn_avx512 && clib_cpu_supports_avx512f ())
acl_out_l2_ip6_node.function = acl_out_ip6_l2_node_fn_avx512;
else if (acl_out_ip6_l2_node_fn_avx2 && clib_cpu_supports_avx2 ())
acl_out_l2_ip6_node.function = acl_out_ip6_l2_node_fn_avx2;
if (acl_in_ip4_fa_node_fn_avx512 && clib_cpu_supports_avx512f ())
acl_in_fa_ip4_node.function = acl_in_ip4_fa_node_fn_avx512;
else if (acl_in_ip4_fa_node_fn_avx2 && clib_cpu_supports_avx2 ())
acl_in_fa_ip4_node.function = acl_in_ip4_fa_node_fn_avx2;
if (acl_out_ip4_fa_node_fn_avx512 && clib_cpu_supports_avx512f ())
acl_out_fa_ip4_node.function = acl_out_ip4_fa_node_fn_avx512;
else if (acl_out_ip4_fa_node_fn_avx2 && clib_cpu_supports_avx2 ())
acl_out_fa_ip4_node.function = acl_out_ip4_fa_node_fn_avx2;
if (acl_in_ip6_fa_node_fn_avx512 && clib_cpu_supports_avx512f ())
acl_in_fa_ip6_node.function = acl_in_ip6_fa_node_fn_avx512;
else if (acl_in_ip6_fa_node_fn_avx2 && clib_cpu_supports_avx2 ())
acl_in_fa_ip6_node.function = acl_in_ip6_fa_node_fn_avx2;
if (acl_out_ip6_fa_node_fn_avx512 && clib_cpu_supports_avx512f ())
acl_out_fa_ip6_node.function = acl_out_ip6_fa_node_fn_avx512;
else if (acl_out_ip6_fa_node_fn_avx2 && clib_cpu_supports_avx2 ())
acl_out_fa_ip6_node.function = acl_out_ip6_fa_node_fn_avx2;
}
#endif
#ifndef CLIB_MULTIARCH_VARIANT
#ifndef CLIB_MARCH_VARIANT
static u8 *
format_fa_5tuple (u8 * s, va_list * args)
{
@@ -549,7 +465,6 @@ static char *acl_fa_error_strings[] = {
VLIB_REGISTER_NODE (acl_in_l2_ip6_node) =
{
.function = acl_in_ip6_l2_node_fn,
.name = "acl-plugin-in-ip6-l2",
.vector_size = sizeof (u32),
.format_trace = format_acl_plugin_trace,
@@ -565,7 +480,6 @@ VLIB_REGISTER_NODE (acl_in_l2_ip6_node) =
VLIB_REGISTER_NODE (acl_in_l2_ip4_node) =
{
.function = acl_in_ip4_l2_node_fn,
.name = "acl-plugin-in-ip4-l2",
.vector_size = sizeof (u32),
.format_trace = format_acl_plugin_trace,
@@ -581,7 +495,6 @@ VLIB_REGISTER_NODE (acl_in_l2_ip4_node) =
VLIB_REGISTER_NODE (acl_out_l2_ip6_node) =
{
.function = acl_out_ip6_l2_node_fn,
.name = "acl-plugin-out-ip6-l2",
.vector_size = sizeof (u32),
.format_trace = format_acl_plugin_trace,
@@ -597,7 +510,6 @@ VLIB_REGISTER_NODE (acl_out_l2_ip6_node) =
VLIB_REGISTER_NODE (acl_out_l2_ip4_node) =
{
.function = acl_out_ip4_l2_node_fn,
.name = "acl-plugin-out-ip4-l2",
.vector_size = sizeof (u32),
.format_trace = format_acl_plugin_trace,
@@ -614,7 +526,6 @@ VLIB_REGISTER_NODE (acl_out_l2_ip4_node) =
VLIB_REGISTER_NODE (acl_in_fa_ip6_node) =
{
.function = acl_in_ip6_fa_node_fn,
.name = "acl-plugin-in-ip6-fa",
.vector_size = sizeof (u32),
.format_trace = format_acl_plugin_trace,
@@ -637,7 +548,6 @@ VNET_FEATURE_INIT (acl_in_ip6_fa_feature, static) =
VLIB_REGISTER_NODE (acl_in_fa_ip4_node) =
{
.function = acl_in_ip4_fa_node_fn,
.name = "acl-plugin-in-ip4-fa",
.vector_size = sizeof (u32),
.format_trace = format_acl_plugin_trace,
@@ -661,7 +571,6 @@ VNET_FEATURE_INIT (acl_in_ip4_fa_feature, static) =
VLIB_REGISTER_NODE (acl_out_fa_ip6_node) =
{
.function = acl_out_ip6_fa_node_fn,
.name = "acl-plugin-out-ip6-fa",
.vector_size = sizeof (u32),
.format_trace = format_acl_plugin_trace,
@@ -684,7 +593,6 @@ VNET_FEATURE_INIT (acl_out_ip6_fa_feature, static) =
VLIB_REGISTER_NODE (acl_out_fa_ip4_node) =
{
.function = acl_out_ip4_fa_node_fn,
.name = "acl-plugin-out-ip4-fa",
.vector_size = sizeof (u32),
.format_trace = format_acl_plugin_trace,
src/plugins/avf.am
+5 -5
View File
@@ -25,7 +25,7 @@ avf_plugin_la_SOURCES = \
noinst_HEADERS += avf/avf.h
if CPU_X86_64
avf_multiversioning_files = \
avf_multiversioning_sources = \
avf/input.c \
avf/output.c
@@ -33,10 +33,10 @@ if CC_SUPPORTS_AVX2
###############################################################
# AVX2
###############################################################
libavf_plugin_avx2_la_SOURCES = $(avf_multiversioning_files)
libavf_plugin_avx2_la_SOURCES = $(avf_multiversioning_sources)
libavf_plugin_avx2_la_CFLAGS = \
$(AM_CFLAGS) @CPU_AVX2_FLAGS@ \
-DCLIB_MULTIARCH_VARIANT=avx2
-DCLIB_MARCH_VARIANT=avx2
noinst_LTLIBRARIES += libavf_plugin_avx2.la
avf_plugin_la_LIBADD += libavf_plugin_avx2.la
endif
@@ -45,10 +45,10 @@ if CC_SUPPORTS_AVX512
###############################################################
# AVX512
###############################################################
libavf_plugin_avx512_la_SOURCES = $(avf_multiversioning_files)
libavf_plugin_avx512_la_SOURCES = $(avf_multiversioning_sources)
libavf_plugin_avx512_la_CFLAGS = \
$(AM_CFLAGS) @CPU_AVX512_FLAGS@ \
-DCLIB_MULTIARCH_VARIANT=avx512
-DCLIB_MARCH_VARIANT=avx512
noinst_LTLIBRARIES += libavf_plugin_avx512.la
avf_plugin_la_LIBADD += libavf_plugin_avx512.la
endif
src/plugins/avf/input.c
+2 -18
View File
@@ -375,8 +375,7 @@ avf_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
return n_rx_packets;
}
uword
CLIB_MULTIARCH_FN (avf_input) (vlib_main_t * vm, vlib_node_runtime_t * node,
VLIB_NODE_FN (avf_input_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
vlib_frame_t * frame)
{
u32 n_rx = 0;
@@ -399,10 +398,9 @@ CLIB_MULTIARCH_FN (avf_input) (vlib_main_t * vm, vlib_node_runtime_t * node,
return n_rx;
}
#ifndef CLIB_MULTIARCH_VARIANT
#ifndef CLIB_MARCH_VARIANT
/* *INDENT-OFF* */
VLIB_REGISTER_NODE (avf_input_node) = {
.function = avf_input,
.name = "avf-input",
.sibling_of = "device-input",
.format_trace = format_avf_input_trace,
@@ -411,20 +409,6 @@ VLIB_REGISTER_NODE (avf_input_node) = {
.n_errors = AVF_INPUT_N_ERROR,
.error_strings = avf_input_error_strings,
};
#if __x86_64__
vlib_node_function_t __clib_weak avf_input_avx512;
vlib_node_function_t __clib_weak avf_input_avx2;
static void __clib_constructor
avf_input_multiarch_select (void)
{
if (avf_input_avx512 && clib_cpu_supports_avx512f ())
avf_input_node.function = avf_input_avx512;
else if (avf_input_avx2 && clib_cpu_supports_avx2 ())
avf_input_node.function = avf_input_avx2;
}
#endif
#endif
/* *INDENT-ON* */
src/plugins/avf/output.c
+1 -1
View File
@@ -159,7 +159,7 @@ CLIB_MULTIARCH_FN (avf_interface_tx) (vlib_main_t * vm,
return frame->n_vectors - n_left;
}
#ifndef CLIB_MULTIARCH_VARIANT
#ifndef CLIB_MARCH_VARIANT
#if __x86_64__
vlib_node_function_t __clib_weak avf_interface_tx_avx512;
vlib_node_function_t __clib_weak avf_interface_tx_avx2;
src/plugins/dpdk.am
+5 -5
View File
@@ -57,7 +57,7 @@ dpdk_plugin_la_SOURCES = \
API_FILES += dpdk/api/dpdk.api
if CPU_X86_64
dpdk_multiversioning_files = \
dpdk_multiversioning_sources = \
dpdk/buffer.c \
dpdk/device/node.c \
dpdk/device/device.c
@@ -66,10 +66,10 @@ if CC_SUPPORTS_AVX2
###############################################################
# AVX2
###############################################################
libdpdk_plugin_avx2_la_SOURCES = $(dpdk_multiversioning_files)
libdpdk_plugin_avx2_la_SOURCES = $(dpdk_multiversioning_sources)
libdpdk_plugin_avx2_la_CFLAGS = \
$(AM_CFLAGS) @CPU_AVX2_FLAGS@ \
-DCLIB_MULTIARCH_VARIANT=avx2
-DCLIB_MARCH_VARIANT=avx2
noinst_LTLIBRARIES += libdpdk_plugin_avx2.la
dpdk_plugin_la_LIBADD += libdpdk_plugin_avx2.la
endif
@@ -78,10 +78,10 @@ if CC_SUPPORTS_AVX512
###############################################################
# AVX512
###############################################################
libdpdk_plugin_avx512_la_SOURCES = $(dpdk_multiversioning_files)
libdpdk_plugin_avx512_la_SOURCES = $(dpdk_multiversioning_sources)
libdpdk_plugin_avx512_la_CFLAGS = \
$(AM_CFLAGS) @CPU_AVX512_FLAGS@ \
-DCLIB_MULTIARCH_VARIANT=avx512
-DCLIB_MARCH_VARIANT=avx512
noinst_LTLIBRARIES += libdpdk_plugin_avx512.la
dpdk_plugin_la_LIBADD += libdpdk_plugin_avx512.la
endif
src/plugins/dpdk/buffer.c
+2 -2
View File
@@ -146,7 +146,7 @@ next:
}
}
#ifndef CLIB_MULTIARCH_VARIANT
#ifndef CLIB_MARCH_VARIANT
static void
del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f)
{
@@ -436,7 +436,7 @@ CLIB_MULTIARCH_FN (dpdk_buffer_free_no_next) (vlib_main_t * vm, u32 * buffers,
0);
}
#ifndef CLIB_MULTIARCH_VARIANT
#ifndef CLIB_MARCH_VARIANT
static void
dpdk_packet_template_init (vlib_main_t * vm,
void *vt,
src/plugins/dpdk/device/device.c
+3 -3
View File
@@ -37,7 +37,7 @@ typedef enum
DPDK_TX_FUNC_N_ERROR,
} dpdk_tx_func_error_t;
#ifndef CLIB_MULTIARCH_VARIANT
#ifndef CLIB_MARCH_VARIANT
static char *dpdk_tx_func_error_strings[] = {
#define _(n,s) s,
foreach_dpdk_tx_func_error
@@ -517,7 +517,7 @@ CLIB_MULTIARCH_FN (dpdk_interface_tx) (vlib_main_t * vm,
return tx_pkts;
}
#ifndef CLIB_MULTIARCH_VARIANT
#ifndef CLIB_MARCH_VARIANT
static void
dpdk_clear_hw_interface_counters (u32 instance)
{
@@ -695,7 +695,7 @@ dpdk_interface_tx_multiarch_select (void)
#define UP_DOWN_FLAG_EVENT 1
#ifndef CLIB_MULTIARCH_VARIANT
#ifndef CLIB_MARCH_VARIANT
uword
admin_up_down_process (vlib_main_t * vm,
vlib_node_runtime_t * rt, vlib_frame_t * f)
src/plugins/dpdk/device/node.c
+3 -19
View File
@@ -28,7 +28,7 @@
#include <dpdk/device/dpdk_priv.h>
#ifndef CLIB_MULTIARCH_VARIANT
#ifndef CLIB_MARCH_VARIANT
static char *dpdk_error_strings[] = {
#define _(n,s) s,
foreach_dpdk_error
@@ -636,8 +636,7 @@ dpdk_device_input (vlib_main_t * vm, dpdk_main_t * dm, dpdk_device_t * xd,
return n_rx_packets;
}
uword CLIB_CPU_OPTIMIZED
CLIB_MULTIARCH_FN (dpdk_input) (vlib_main_t * vm, vlib_node_runtime_t * node,
VLIB_NODE_FN (dpdk_input_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
vlib_frame_t * f)
{
dpdk_main_t *dm = &dpdk_main;
@@ -666,10 +665,9 @@ CLIB_MULTIARCH_FN (dpdk_input) (vlib_main_t * vm, vlib_node_runtime_t * node,
return n_rx_packets;
}
#ifndef CLIB_MULTIARCH_VARIANT
#ifndef CLIB_MARCH_VARIANT
/* *INDENT-OFF* */
VLIB_REGISTER_NODE (dpdk_input_node) = {
.function = dpdk_input,
.type = VLIB_NODE_TYPE_INPUT,
.name = "dpdk-input",
.sibling_of = "device-input",
@@ -684,20 +682,6 @@ VLIB_REGISTER_NODE (dpdk_input_node) = {
.error_strings = dpdk_error_strings,
};
/* *INDENT-ON* */
vlib_node_function_t __clib_weak dpdk_input_avx512;
vlib_node_function_t __clib_weak dpdk_input_avx2;
#if __x86_64__
static void __clib_constructor
dpdk_input_multiarch_select (void)
{
if (dpdk_input_avx512 && clib_cpu_supports_avx512f ())
dpdk_input_node.function = dpdk_input_avx512;
else if (dpdk_input_avx2 && clib_cpu_supports_avx2 ())
dpdk_input_node.function = dpdk_input_avx2;
}
#endif
#endif
/*
src/plugins/memif.am
+5 -5
View File
@@ -36,7 +36,7 @@ nobase_apiinclude_HEADERS += \
API_FILES += memif/memif.api
if CPU_X86_64
memif_multiversioning_files = \
memif_multiversioning_sources = \
memif/node.c \
memif/device.c
@@ -44,10 +44,10 @@ if CC_SUPPORTS_AVX2
###############################################################
# AVX2
###############################################################
libmemif_plugin_avx2_la_SOURCES = $(memif_multiversioning_files)
libmemif_plugin_avx2_la_SOURCES = $(memif_multiversioning_sources)
libmemif_plugin_avx2_la_CFLAGS = \
$(AM_CFLAGS) @CPU_AVX2_FLAGS@ \
-DCLIB_MULTIARCH_VARIANT=avx2
-DCLIB_MARCH_VARIANT=avx2
noinst_LTLIBRARIES += libmemif_plugin_avx2.la
memif_plugin_la_LIBADD += libmemif_plugin_avx2.la
endif
@@ -56,10 +56,10 @@ if CC_SUPPORTS_AVX512
###############################################################
# AVX512
###############################################################
libmemif_plugin_avx512_la_SOURCES = $(memif_multiversioning_files)
libmemif_plugin_avx512_la_SOURCES = $(memif_multiversioning_sources)
libmemif_plugin_avx512_la_CFLAGS = \
$(AM_CFLAGS) @CPU_AVX512_FLAGS@ \
-DCLIB_MULTIARCH_VARIANT=avx512
-DCLIB_MARCH_VARIANT=avx512
noinst_LTLIBRARIES += libmemif_plugin_avx512.la
memif_plugin_la_LIBADD += libmemif_plugin_avx512.la
endif
src/plugins/memif/device.c
+2 -2
View File
@@ -46,7 +46,7 @@ static __clib_unused char *memif_tx_func_error_strings[] = {
#undef _
};
#ifndef CLIB_MULTIARCH_VARIANT
#ifndef CLIB_MARCH_VARIANT
u8 *
format_memif_device_name (u8 * s, va_list * args)
{
@@ -500,7 +500,7 @@ memif_subif_add_del_function (vnet_main_t * vnm,
return 0;
}
#ifndef CLIB_MULTIARCH_VARIANT
#ifndef CLIB_MARCH_VARIANT
/* *INDENT-OFF* */
VNET_DEVICE_CLASS (memif_device_class) = {
.name = "memif",
src/plugins/memif/node.c
+5 -20
View File
@@ -854,10 +854,10 @@ done:
return n_rx_packets;
}
uword
CLIB_MULTIARCH_FN (memif_input_fn) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
VLIB_NODE_FN (memif_input_node) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
{
u32 n_rx = 0;
memif_main_t *mm = &memif_main;
@@ -910,10 +910,9 @@ CLIB_MULTIARCH_FN (memif_input_fn) (vlib_main_t * vm,
return n_rx;
}
#ifndef CLIB_MULTIARCH_VARIANT
#ifndef CLIB_MARCH_VARIANT
/* *INDENT-OFF* */
VLIB_REGISTER_NODE (memif_input_node) = {
.function = memif_input_fn,
.name = "memif-input",
.sibling_of = "device-input",
.format_trace = format_memif_input_trace,
@@ -922,20 +921,6 @@ VLIB_REGISTER_NODE (memif_input_node) = {
.n_errors = MEMIF_INPUT_N_ERROR,
.error_strings = memif_input_error_strings,
};
vlib_node_function_t __clib_weak memif_input_fn_avx512;
vlib_node_function_t __clib_weak memif_input_fn_avx2;
#if __x86_64__
static void __clib_constructor
memif_input_multiarch_select (void)
{
if (memif_input_fn_avx512 && clib_cpu_supports_avx512f ())
memif_input_node.function = memif_input_fn_avx512;
else if (memif_input_fn_avx2 && clib_cpu_supports_avx2 ())
memif_input_node.function = memif_input_fn_avx2;
}
#endif
#endif
/* *INDENT-ON* */
src/vlib/node.c
+20
View File
@@ -324,6 +324,26 @@ register_node (vlib_main_t * vm, vlib_node_registration_t * r)
ASSERT (VLIB_NODE_TYPE_INTERNAL == zero.type);
}
if (r->node_fn_registrations)
{
vlib_node_fn_registration_t *fnr = r->node_fn_registrations;
int priority = -1;
/* to avoid confusion, please remove ".function " statiement from
CLIB_NODE_REGISTRATION() if using function function candidates */
ASSERT (r->function == 0);
while (fnr)
{
if (fnr->priority > priority)
{
priority = fnr->priority;
r->function = fnr->function;
}
fnr = fnr->next_registration;
}
}
ASSERT (r->function != 0);
n = clib_mem_alloc_no_fail (sizeof (n[0]));
src/vlib/node.h
+28
View File
@@ -75,11 +75,21 @@ typedef enum
VLIB_N_NODE_TYPE,
} vlib_node_type_t;
typedef struct _vlib_node_fn_registration
{
vlib_node_function_t *function;
int priority;
struct _vlib_node_fn_registration *next_registration;
} vlib_node_fn_registration_t;
typedef struct _vlib_node_registration
{
/* Vector processing function for this node. */
vlib_node_function_t *function;
/* Node function candidate registration with priority */
vlib_node_fn_registration_t *node_fn_registrations;
/* Node name. */
char *name;
@@ -160,6 +170,24 @@ static void __vlib_rm_node_registration_##x (void) \
} \
__VA_ARGS__ vlib_node_registration_t x
#define VLIB_NODE_FN(node) \
uword CLIB_MARCH_SFX (node##_fn)(); \
static vlib_node_fn_registration_t \
CLIB_MARCH_SFX(node##_fn_registration) = \
{ .function = &CLIB_MARCH_SFX (node##_fn), }; \
\
static void __clib_constructor \
CLIB_MARCH_SFX (node##_multiarch_register) (void) \
{ \
extern vlib_node_registration_t node; \
vlib_node_fn_registration_t *r; \
r = & CLIB_MARCH_SFX (node##_fn_registration); \
r->priority = CLIB_MARCH_FN_PRIORITY(); \
r->next_registration = node.node_fn_registrations; \
node.node_fn_registrations = r; \
} \
uword CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (node##_fn)
#if CLIB_DEBUG > 0
#define VLIB_NODE_FUNCTION_CLONE_TEMPLATE(arch, fn)
#define VLIB_NODE_FUNCTION_MULTIARCH_CLONE(fn)
src/vnet.am
+8 -8
View File
@@ -21,7 +21,7 @@ libvnet_la_DEPENDENCIES = \
libsvmdb.la \
libsvm.la \
libvlibmemory.la
libvnet_multiversioning_files =
libvnet_multiversioning_sources =
libvnet_la_LIBADD = $(libvnet_la_DEPENDENCIES) -lm -lpthread -ldl -lrt
@@ -128,7 +128,7 @@ libvnet_la_SOURCES += \
vnet/ethernet/p2p_ethernet_input.c \
vnet/ethernet/p2p_ethernet_api.c
libvnet_multiversioning_files += \
libvnet_multiversioning_sources += \
vnet/l2/l2_output.c
nobase_include_HEADERS += \
@@ -294,7 +294,7 @@ nobase_include_HEADERS += \
vnet/bonding/node.h \
vnet/bonding/bond.api.h
libvnet_multiversioning_files = \
libvnet_multiversioning_sources += \
vnet/bonding/node.c
API_FILES += vnet/bonding/bond.api
@@ -416,7 +416,7 @@ API_FILES += \
vnet/ip/rd_cp.api \
vnet/ip/punt.api
libvnet_multiversioning_files = \
libvnet_multiversioning_sources += \
vnet/ip/ip4_input.c
########################################
@@ -1286,10 +1286,10 @@ if CC_SUPPORTS_AVX2
###############################################################
# AVX2
###############################################################
libvnet_avx2_la_SOURCES = $(libvnet_multiversioning_files)
libvnet_avx2_la_SOURCES = $(libvnet_multiversioning_sources)
libvnet_avx2_la_CFLAGS = \
$(AM_CFLAGS) @CPU_AVX2_FLAGS@ \
-DCLIB_MULTIARCH_VARIANT=avx2
-DCLIB_MARCH_VARIANT=avx2
noinst_LTLIBRARIES += libvnet_avx2.la
libvnet_la_LIBADD += libvnet_avx2.la
endif
@@ -1298,10 +1298,10 @@ if CC_SUPPORTS_AVX512
###############################################################
# AVX512
###############################################################
libvnet_avx512_la_SOURCES = $(libvnet_multiversioning_files)
libvnet_avx512_la_SOURCES = $(libvnet_multiversioning_sources)
libvnet_avx512_la_CFLAGS = \
$(AM_CFLAGS) @CPU_AVX512_FLAGS@ \
-DCLIB_MULTIARCH_VARIANT=avx512
-DCLIB_MARCH_VARIANT=avx512
noinst_LTLIBRARIES += libvnet_avx512.la
libvnet_la_LIBADD += libvnet_avx512.la
endif
src/vnet/bonding/node.c
+5 -21
View File
@@ -36,7 +36,7 @@ typedef enum
BOND_INPUT_N_ERROR,
} bond_input_error_t;
#ifndef CLIB_MULTIARCH_VARIANT
#ifndef CLIB_MARCH_VARIANT
static char *bond_input_error_strings[] = {
#define _(n,s) s,
foreach_bond_input_error
@@ -172,10 +172,9 @@ bond_update_next (vlib_main_t * vm, vlib_node_runtime_t * node,
vnet_feature_next ( /* not used */ 0, next_index, b);
}
uword CLIB_CPU_OPTIMIZED
CLIB_MULTIARCH_FN (bond_input_fn) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
VLIB_NODE_FN (bond_input_node) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
{
u16 thread_index = vlib_get_thread_index ();
u32 *from, n_left;
@@ -367,7 +366,7 @@ CLIB_MULTIARCH_FN (bond_input_fn) (vlib_main_t * vm,
return frame->n_vectors;
}
#ifndef CLIB_MULTIARCH_VARIANT
#ifndef CLIB_MARCH_VARIANT
static clib_error_t *
bond_input_init (vlib_main_t * vm)
{
@@ -376,7 +375,6 @@ bond_input_init (vlib_main_t * vm)
/* *INDENT-OFF* */
VLIB_REGISTER_NODE (bond_input_node) = {
.function = bond_input_fn,
.name = "bond-input",
.vector_size = sizeof (u32),
.format_buffer = format_ethernet_header_with_length,
@@ -391,20 +389,6 @@ VLIB_REGISTER_NODE (bond_input_node) = {
}
};
#if __x86_64__
vlib_node_function_t __clib_weak bond_input_fn_avx512;
vlib_node_function_t __clib_weak bond_input_fn_avx2;
static void __clib_constructor
bond_input_multiarch_select (void)
{
if (bond_input_fn_avx512 && clib_cpu_supports_avx512f ())
bond_input_node.function = bond_input_fn_avx512;
else if (bond_input_fn_avx2 && clib_cpu_supports_avx2 ())
bond_input_node.function = bond_input_fn_avx2;
}
#endif
VLIB_INIT_FUNCTION (bond_input_init);
VNET_FEATURE_INIT (bond_input, static) =
src/vnet/ip/ip4_input.c
+5 -29
View File
@@ -47,7 +47,7 @@ typedef struct
u8 packet_data[64];
} ip4_input_trace_t;
#ifndef CLIB_MULTIARCH_VARIANT
#ifndef CLIB_MARCH_VARIANT
static u8 *
format_ip4_input_trace (u8 * s, va_list * va)
{
@@ -292,22 +292,20 @@ ip4_input_inline (vlib_main_t * vm,
<code> vnet_get_config_data (... &next0 ...); </code>
or @c error-drop
*/
uword CLIB_CPU_OPTIMIZED
CLIB_MULTIARCH_FN (ip4_input) (vlib_main_t * vm, vlib_node_runtime_t * node,
VLIB_NODE_FN (ip4_input_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
vlib_frame_t * frame)
{
return ip4_input_inline (vm, node, frame, /* verify_checksum */ 1);
}
uword CLIB_CPU_OPTIMIZED
CLIB_MULTIARCH_FN (ip4_input_no_checksum) (vlib_main_t * vm,
VLIB_NODE_FN (ip4_input_no_checksum_node) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
{
return ip4_input_inline (vm, node, frame, /* verify_checksum */ 0);
}
#ifndef CLIB_MULTIARCH_VARIANT
#ifndef CLIB_MARCH_VARIANT
char *ip4_error_strings[] = {
#define _(sym,string) string,
foreach_ip4_error
@@ -316,7 +314,6 @@ char *ip4_error_strings[] = {
/* *INDENT-OFF* */
VLIB_REGISTER_NODE (ip4_input_node) = {
.function = ip4_input,
.name = "ip4-input",
.vector_size = sizeof (u32),
@@ -337,8 +334,7 @@ VLIB_REGISTER_NODE (ip4_input_node) = {
.format_trace = format_ip4_input_trace,
};
VLIB_REGISTER_NODE (ip4_input_no_checksum_node,static) = {
.function = ip4_input_no_checksum,
VLIB_REGISTER_NODE (ip4_input_no_checksum_node) = {
.name = "ip4-input-no-checksum",
.vector_size = sizeof (u32),
@@ -357,26 +353,6 @@ VLIB_REGISTER_NODE (ip4_input_no_checksum_node,static) = {
};
/* *INDENT-ON* */
#if __x86_64__
vlib_node_function_t __clib_weak ip4_input_avx512;
vlib_node_function_t __clib_weak ip4_input_avx2;
vlib_node_function_t __clib_weak ip4_input_no_checksum_avx512;
vlib_node_function_t __clib_weak ip4_input_no_checksum_avx2;
static void __clib_constructor
ip4_input_multiarch_select (void)
{
if (ip4_input_no_checksum_avx512 && clib_cpu_supports_avx512f ())
ip4_input_no_checksum_node.function = ip4_input_no_checksum_avx512;
else if (ip4_input_no_checksum_avx2 && clib_cpu_supports_avx2 ())
ip4_input_no_checksum_node.function = ip4_input_no_checksum_avx2;
if (ip4_input_avx512 && clib_cpu_supports_avx512f ())
ip4_input_node.function = ip4_input_avx512;
else if (ip4_input_avx2 && clib_cpu_supports_avx2 ())
ip4_input_node.function = ip4_input_avx2;
}
#endif
static clib_error_t *
ip4_init (vlib_main_t * vm)
{
src/vnet/l2/l2_output.c
+6 -22
View File
@@ -27,7 +27,7 @@
#include <vnet/l2/l2_output.h>
#ifndef CLIB_MULTIARCH_VARIANT
#ifndef CLIB_MARCH_VARIANT
/* Feature graph node names */
static char *l2output_feat_names[] = {
#define _(sym,name) name,
@@ -76,7 +76,7 @@ typedef struct
u8 raw[12]; /* raw data */
} l2output_trace_t;
#ifndef CLIB_MULTIARCH_VARIANT
#ifndef CLIB_MARCH_VARIANT
/* packet trace format function */
static u8 *
format_l2output_trace (u8 * s, va_list * args)
@@ -275,10 +275,9 @@ l2output_process_batch (vlib_main_t * vm, vlib_node_runtime_t * node,
l2_efp, l2_vtr, l2_pbb, 1, 1);
}
uword CLIB_CPU_OPTIMIZED
CLIB_MULTIARCH_FN (l2output_node_fn) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
VLIB_NODE_FN (l2output_node) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
{
u32 n_left, *from;
l2output_main_t *msm = &l2output_main;
@@ -431,10 +430,9 @@ CLIB_MULTIARCH_FN (l2output_node_fn) (vlib_main_t * vm,
return frame->n_vectors;
}
#ifndef CLIB_MULTIARCH_VARIANT
#ifndef CLIB_MARCH_VARIANT
/* *INDENT-OFF* */
VLIB_REGISTER_NODE (l2output_node) = {
.function = l2output_node_fn,
.name = "l2-output",
.vector_size = sizeof (u32),
.format_trace = format_l2output_trace,
@@ -451,20 +449,6 @@ VLIB_REGISTER_NODE (l2output_node) = {
[L2OUTPUT_NEXT_BAD_INTF] = "l2-output-bad-intf",
},
};
#if __x86_64__
vlib_node_function_t __clib_weak l2output_node_fn_avx512;
vlib_node_function_t __clib_weak l2output_node_fn_avx2;
static void __clib_constructor
l2output_multiarch_select (void)
{
if (l2output_node_fn_avx512 && clib_cpu_supports_avx512f ())
l2output_node.function = l2output_node_fn_avx512;
else if (l2output_node_fn_avx2 && clib_cpu_supports_avx2 ())
l2output_node.function = l2output_node_fn_avx2;
}
#endif
/* *INDENT-ON* */
src/vppinfra/cpu.h
+25 -2
View File
@@ -51,14 +51,16 @@
return & fn; \
}
#ifdef CLIB_MULTIARCH_VARIANT
#ifdef CLIB_MARCH_VARIANT
#define __CLIB_MULTIARCH_FN(a,b) a##_##b
#define _CLIB_MULTIARCH_FN(a,b) __CLIB_MULTIARCH_FN(a,b)
#define CLIB_MULTIARCH_FN(fn) _CLIB_MULTIARCH_FN(fn,CLIB_MULTIARCH_VARIANT)
#define CLIB_MULTIARCH_FN(fn) _CLIB_MULTIARCH_FN(fn,CLIB_MARCH_VARIANT)
#else
#define CLIB_MULTIARCH_FN(fn) fn
#endif
#define CLIB_MARCH_SFX CLIB_MULTIARCH_FN
#define foreach_x86_64_flags \
_ (sse3, 1, ecx, 0) \
_ (ssse3, 1, ecx, 9) \
@@ -165,6 +167,27 @@ clib_cpu_supports_aes ()
#endif
}
static inline int
clib_cpu_march_priority_avx512 ()
{
if (clib_cpu_supports_avx512f ())
return 20;
return -1;
}
static inline int
clib_cpu_march_priority_avx2 ()
{
if (clib_cpu_supports_avx2 ())
return 10;
return -1;
}
#ifdef CLIB_MARCH_VARIANT
#define CLIB_MARCH_FN_PRIORITY() CLIB_MARCH_SFX(clib_cpu_march_priority)()
#else
#define CLIB_MARCH_FN_PRIORITY() 0
#endif
#endif /* included_clib_cpu_h */
format_function_t format_cpu_uarch;