dpdk: complete rework of the dpdk-input node
Change-Id: If174d189de40e6f9ffae99997bba93a2519d9fda Signed-off-by: Damjan Marion <damarion@cisco.com>
This commit is contained in:
Damjan Marion
committed by
Dave Barach
Dave Barach
parent
4a6d02337b
commit
c576622667
@@ -43,7 +43,7 @@ dpdk_device_setup (dpdk_device_t * xd)
|
||||
{
|
||||
dpdk_main_t *dm = &dpdk_main;
|
||||
vnet_main_t *vnm = vnet_get_main ();
|
||||
vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, xd->vlib_sw_if_index);
|
||||
vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, xd->sw_if_index);
|
||||
vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, xd->hw_if_index);
|
||||
int rv;
|
||||
int j;
|
||||
|
||||
@@ -117,7 +117,7 @@ dpdk_tx_trace_buffer (dpdk_main_t * dm,
|
||||
u16 queue_id, u32 buffer_index, vlib_buffer_t * buffer)
|
||||
{
|
||||
vlib_main_t *vm = vlib_get_main ();
|
||||
dpdk_tx_dma_trace_t *t0;
|
||||
dpdk_tx_trace_t *t0;
|
||||
struct rte_mbuf *mb;
|
||||
|
||||
mb = rte_mbuf_from_vlib_buffer (buffer);
|
||||
@@ -608,7 +608,7 @@ CLIB_MULTIARCH_FN (dpdk_interface_tx) (vlib_main_t * vm,
|
||||
cm = vec_elt_at_index (vnm->interface_main.sw_if_counters,
|
||||
VNET_INTERFACE_COUNTER_TX_ERROR);
|
||||
|
||||
vlib_increment_simple_counter (cm, my_cpu, xd->vlib_sw_if_index,
|
||||
vlib_increment_simple_counter (cm, my_cpu, xd->sw_if_index,
|
||||
n_packets);
|
||||
|
||||
vlib_error_count (vm, node->node_index, DPDK_TX_FUNC_ERROR_PKT_DROP,
|
||||
@@ -787,7 +787,7 @@ VNET_DEVICE_CLASS (dpdk_device_class) = {
|
||||
.tx_function_error_strings = dpdk_tx_func_error_strings,
|
||||
.format_device_name = format_dpdk_device_name,
|
||||
.format_device = format_dpdk_device,
|
||||
.format_tx_trace = format_dpdk_tx_dma_trace,
|
||||
.format_tx_trace = format_dpdk_tx_trace,
|
||||
.clear_counters = dpdk_clear_hw_interface_counters,
|
||||
.admin_up_down_function = dpdk_interface_admin_up_down,
|
||||
.subif_add_del_function = dpdk_subif_add_del_function,
|
||||
|
||||
@@ -181,14 +181,13 @@ typedef struct
|
||||
dpdk_portid_t device_index;
|
||||
|
||||
u32 hw_if_index;
|
||||
u32 vlib_sw_if_index;
|
||||
u32 sw_if_index;
|
||||
|
||||
/* next node index if we decide to steal the rx graph arc */
|
||||
u32 per_interface_next_index;
|
||||
|
||||
/* dpdk rte_mbuf rx and tx vectors, VLIB_FRAME_SIZE */
|
||||
struct rte_mbuf ***tx_vectors; /* one per worker thread */
|
||||
struct rte_mbuf ***rx_vectors;
|
||||
|
||||
dpdk_pmd_t pmd:8;
|
||||
i8 cpu_socket;
|
||||
@@ -350,19 +349,30 @@ typedef struct
|
||||
|
||||
extern dpdk_config_main_t dpdk_config_main;
|
||||
|
||||
#define DPDK_RX_BURST_SZ VLIB_FRAME_SIZE
|
||||
|
||||
typedef struct
|
||||
{
|
||||
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
|
||||
struct rte_mbuf *mbufs[DPDK_RX_BURST_SZ];
|
||||
u32 buffers[DPDK_RX_BURST_SZ];
|
||||
u16 next[DPDK_RX_BURST_SZ];
|
||||
u16 etype[DPDK_RX_BURST_SZ];
|
||||
u8 flags[DPDK_RX_BURST_SZ];
|
||||
vlib_buffer_t buffer_template;
|
||||
} dpdk_per_thread_data_t;
|
||||
|
||||
typedef struct
|
||||
{
|
||||
|
||||
/* Devices */
|
||||
dpdk_device_t *devices;
|
||||
dpdk_device_and_queue_t **devices_by_hqos_cpu;
|
||||
dpdk_per_thread_data_t *per_thread_data;
|
||||
|
||||
/* per-thread recycle lists */
|
||||
u32 **recycle;
|
||||
|
||||
/* per-thread buffer templates */
|
||||
vlib_buffer_t *buffer_templates;
|
||||
|
||||
/* buffer flags template, configurable to enable/disable tcp / udp cksum */
|
||||
u32 buffer_flags_template;
|
||||
|
||||
@@ -416,7 +426,7 @@ typedef struct
|
||||
/* Copy of VLIB buffer; packet data stored in pre_data. */
|
||||
vlib_buffer_t buffer;
|
||||
u8 data[256]; /* First 256 data bytes, used for hexdump */
|
||||
} dpdk_tx_dma_trace_t;
|
||||
} dpdk_tx_trace_t;
|
||||
|
||||
typedef struct
|
||||
{
|
||||
@@ -426,7 +436,7 @@ typedef struct
|
||||
struct rte_mbuf mb;
|
||||
vlib_buffer_t buffer; /* Copy of VLIB buffer; pkt data stored in pre_data. */
|
||||
u8 data[256]; /* First 256 data bytes, used for hexdump */
|
||||
} dpdk_rx_dma_trace_t;
|
||||
} dpdk_rx_trace_t;
|
||||
|
||||
void dpdk_device_setup (dpdk_device_t * xd);
|
||||
void dpdk_device_start (dpdk_device_t * xd);
|
||||
@@ -467,8 +477,8 @@ void dpdk_update_link_state (dpdk_device_t * xd, f64 now);
|
||||
format_function_t format_dpdk_device_name;
|
||||
format_function_t format_dpdk_device;
|
||||
format_function_t format_dpdk_device_errors;
|
||||
format_function_t format_dpdk_tx_dma_trace;
|
||||
format_function_t format_dpdk_rx_dma_trace;
|
||||
format_function_t format_dpdk_tx_trace;
|
||||
format_function_t format_dpdk_rx_trace;
|
||||
format_function_t format_dpdk_rte_mbuf;
|
||||
format_function_t format_dpdk_rx_rte_mbuf;
|
||||
unformat_function_t unformat_dpdk_log_level;
|
||||
|
||||
@@ -106,7 +106,7 @@ dpdk_update_counters (dpdk_device_t * xd, f64 now)
|
||||
cm = vec_elt_at_index (vnm->interface_main.sw_if_counters,
|
||||
VNET_INTERFACE_COUNTER_RX_NO_BUF);
|
||||
|
||||
vlib_increment_simple_counter (cm, thread_index, xd->vlib_sw_if_index,
|
||||
vlib_increment_simple_counter (cm, thread_index, xd->sw_if_index,
|
||||
xd->stats.rx_nombuf -
|
||||
xd->last_stats.rx_nombuf);
|
||||
}
|
||||
@@ -117,7 +117,7 @@ dpdk_update_counters (dpdk_device_t * xd, f64 now)
|
||||
cm = vec_elt_at_index (vnm->interface_main.sw_if_counters,
|
||||
VNET_INTERFACE_COUNTER_RX_MISS);
|
||||
|
||||
vlib_increment_simple_counter (cm, thread_index, xd->vlib_sw_if_index,
|
||||
vlib_increment_simple_counter (cm, thread_index, xd->sw_if_index,
|
||||
xd->stats.imissed -
|
||||
xd->last_stats.imissed);
|
||||
}
|
||||
@@ -129,7 +129,7 @@ dpdk_update_counters (dpdk_device_t * xd, f64 now)
|
||||
cm = vec_elt_at_index (vnm->interface_main.sw_if_counters,
|
||||
VNET_INTERFACE_COUNTER_RX_ERROR);
|
||||
|
||||
vlib_increment_simple_counter (cm, thread_index, xd->vlib_sw_if_index,
|
||||
vlib_increment_simple_counter (cm, thread_index, xd->sw_if_index,
|
||||
rxerrors - last_rxerrors);
|
||||
}
|
||||
|
||||
|
||||
@@ -608,16 +608,16 @@ format_dpdk_device (u8 * s, va_list * args)
|
||||
}
|
||||
|
||||
u8 *
|
||||
format_dpdk_tx_dma_trace (u8 * s, va_list * va)
|
||||
format_dpdk_tx_trace (u8 * s, va_list * va)
|
||||
{
|
||||
CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
|
||||
CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
|
||||
CLIB_UNUSED (vnet_main_t * vnm) = vnet_get_main ();
|
||||
dpdk_tx_dma_trace_t *t = va_arg (*va, dpdk_tx_dma_trace_t *);
|
||||
dpdk_tx_trace_t *t = va_arg (*va, dpdk_tx_trace_t *);
|
||||
dpdk_main_t *dm = &dpdk_main;
|
||||
dpdk_device_t *xd = vec_elt_at_index (dm->devices, t->device_index);
|
||||
u32 indent = format_get_indent (s);
|
||||
vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, xd->vlib_sw_if_index);
|
||||
vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, xd->sw_if_index);
|
||||
|
||||
s = format (s, "%U tx queue %d",
|
||||
format_vnet_sw_interface_name, vnm, sw, t->queue_index);
|
||||
@@ -638,17 +638,17 @@ format_dpdk_tx_dma_trace (u8 * s, va_list * va)
|
||||
}
|
||||
|
||||
u8 *
|
||||
format_dpdk_rx_dma_trace (u8 * s, va_list * va)
|
||||
format_dpdk_rx_trace (u8 * s, va_list * va)
|
||||
{
|
||||
CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
|
||||
CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
|
||||
CLIB_UNUSED (vnet_main_t * vnm) = vnet_get_main ();
|
||||
dpdk_rx_dma_trace_t *t = va_arg (*va, dpdk_rx_dma_trace_t *);
|
||||
dpdk_rx_trace_t *t = va_arg (*va, dpdk_rx_trace_t *);
|
||||
dpdk_main_t *dm = &dpdk_main;
|
||||
dpdk_device_t *xd = vec_elt_at_index (dm->devices, t->device_index);
|
||||
format_function_t *f;
|
||||
u32 indent = format_get_indent (s);
|
||||
vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, xd->vlib_sw_if_index);
|
||||
vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, xd->sw_if_index);
|
||||
|
||||
s = format (s, "%U rx queue %d",
|
||||
format_vnet_sw_interface_name, vnm, sw, t->queue_index);
|
||||
|
||||
@@ -239,17 +239,17 @@ dpdk_lib_init (dpdk_main_t * dm)
|
||||
| VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
|
||||
|
||||
/* vlib_buffer_t template */
|
||||
vec_validate_aligned (dm->buffer_templates, tm->n_vlib_mains - 1,
|
||||
vec_validate_aligned (dm->per_thread_data, tm->n_vlib_mains - 1,
|
||||
CLIB_CACHE_LINE_BYTES);
|
||||
for (i = 0; i < tm->n_vlib_mains; i++)
|
||||
{
|
||||
vlib_buffer_free_list_t *fl;
|
||||
vlib_buffer_t *bt = vec_elt_at_index (dm->buffer_templates, i);
|
||||
dpdk_per_thread_data_t *ptd = vec_elt_at_index (dm->per_thread_data, i);
|
||||
fl = vlib_buffer_get_free_list (vm,
|
||||
VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
|
||||
vlib_buffer_init_for_free_list (bt, fl);
|
||||
bt->flags = dm->buffer_flags_template;
|
||||
vnet_buffer (bt)->sw_if_index[VLIB_TX] = (u32) ~ 0;
|
||||
vlib_buffer_init_for_free_list (&ptd->buffer_template, fl);
|
||||
ptd->buffer_template.flags = dm->buffer_flags_template;
|
||||
vnet_buffer (&ptd->buffer_template)->sw_if_index[VLIB_TX] = (u32) ~ 0;
|
||||
}
|
||||
|
||||
for (i = 0; i < nports; i++)
|
||||
@@ -546,15 +546,6 @@ dpdk_lib_init (dpdk_main_t * dm)
|
||||
vec_reset_length (xd->tx_vectors[j]);
|
||||
}
|
||||
|
||||
vec_validate_aligned (xd->rx_vectors, xd->rx_q_used,
|
||||
CLIB_CACHE_LINE_BYTES);
|
||||
for (j = 0; j < xd->rx_q_used; j++)
|
||||
{
|
||||
vec_validate_aligned (xd->rx_vectors[j], VLIB_FRAME_SIZE - 1,
|
||||
CLIB_CACHE_LINE_BYTES);
|
||||
vec_reset_length (xd->rx_vectors[j]);
|
||||
}
|
||||
|
||||
/* count the number of descriptors used for this device */
|
||||
nb_desc += xd->nb_rx_desc + xd->nb_tx_desc * xd->tx_q_used;
|
||||
|
||||
@@ -621,7 +612,7 @@ dpdk_lib_init (dpdk_main_t * dm)
|
||||
xd->port_conf.rxmode.max_rx_pkt_len = max_rx_frame;
|
||||
|
||||
sw = vnet_get_hw_sw_interface (dm->vnet_main, xd->hw_if_index);
|
||||
xd->vlib_sw_if_index = sw->sw_if_index;
|
||||
xd->sw_if_index = sw->sw_if_index;
|
||||
vnet_hw_interface_set_input_node (dm->vnet_main, xd->hw_if_index,
|
||||
dpdk_input_node.index);
|
||||
|
||||
@@ -1398,7 +1389,7 @@ dpdk_update_link_state (dpdk_device_t * xd, f64 now)
|
||||
u8 new_link_state;
|
||||
} *ed;
|
||||
ed = ELOG_DATA (&vm->elog_main, e);
|
||||
ed->sw_if_index = xd->vlib_sw_if_index;
|
||||
ed->sw_if_index = xd->sw_if_index;
|
||||
ed->admin_up = (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) != 0;
|
||||
ed->old_link_state = (u8)
|
||||
vnet_hw_interface_is_link_up (vnm, xd->hw_if_index);
|
||||
@@ -1494,7 +1485,7 @@ dpdk_update_link_state (dpdk_device_t * xd, f64 now)
|
||||
u32 flags;
|
||||
} *ed;
|
||||
ed = ELOG_DATA (&vm->elog_main, e);
|
||||
ed->sw_if_index = xd->vlib_sw_if_index;
|
||||
ed->sw_if_index = xd->sw_if_index;
|
||||
ed->flags = hw_flags;
|
||||
}
|
||||
vnet_hw_interface_set_flags (vnm, xd->hw_if_index, hw_flags);
|
||||
@@ -1607,8 +1598,7 @@ dpdk_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f)
|
||||
(bhi->bond_info, sdev->hw_if_index, 1);
|
||||
/* Set MACs and slave link flags on slave interface */
|
||||
shi = vnet_get_hw_interface (vnm, sdev->hw_if_index);
|
||||
ssi = vnet_get_sw_interface
|
||||
(vnm, sdev->vlib_sw_if_index);
|
||||
ssi = vnet_get_sw_interface (vnm, sdev->sw_if_index);
|
||||
sei = pool_elt_at_index
|
||||
(em->interfaces, shi->hw_instance);
|
||||
shi->bond_info = VNET_HW_INTERFACE_BOND_INFO_SLAVE;
|
||||
|
||||
+516
-425
File diff suppressed because it is too large
Load Diff
@@ -261,6 +261,8 @@ nobase_include_HEADERS = \
|
||||
vppinfra/vec_bootstrap.h \
|
||||
vppinfra/vector.h \
|
||||
vppinfra/vector_altivec.h \
|
||||
vppinfra/vector_avx2.h \
|
||||
vppinfra/vector_avx512.h \
|
||||
vppinfra/vector_funcs.h \
|
||||
vppinfra/vector_neon.h \
|
||||
vppinfra/vector_sse42.h \
|
||||
|
||||
+14
-3
@@ -58,8 +58,11 @@
|
||||
#define CLIB_HAVE_VEC128
|
||||
#endif
|
||||
|
||||
#if defined (__AVX__)
|
||||
#if defined (__AVX2__)
|
||||
#define CLIB_HAVE_VEC256
|
||||
#if defined (__clang__) && __clang_major__ < 4
|
||||
#undef CLIB_HAVE_VEC256
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if defined (__AVX512F__)
|
||||
@@ -179,7 +182,7 @@ t##s##x##c##_sub (t##s##x##c v1, t##s##x##c v2) \
|
||||
foreach_vec
|
||||
#undef _
|
||||
|
||||
/* this macro generate _splat inline funcitons for each scalar vector type */
|
||||
/* this macro generate _splat inline functions for each scalar vector type */
|
||||
#define _(t, s, c) \
|
||||
static_always_inline t##s##x##c \
|
||||
t##s##x##c##_splat (t##s x) \
|
||||
@@ -192,13 +195,21 @@ t##s##x##c##_splat (t##s x) \
|
||||
\
|
||||
return r; \
|
||||
}
|
||||
foreach_int_vec foreach_uint_vec
|
||||
foreach_vec128i foreach_vec128u
|
||||
#undef _
|
||||
|
||||
#if defined (__SSE4_2__) && __GNUC__ >= 4
|
||||
#include <vppinfra/vector_sse42.h>
|
||||
#endif
|
||||
|
||||
#if defined (__AVX2__)
|
||||
#include <vppinfra/vector_avx2.h>
|
||||
#endif
|
||||
|
||||
#if defined (__AVX512F__)
|
||||
#include <vppinfra/vector_avx512.h>
|
||||
#endif
|
||||
|
||||
#if defined (__ALTIVEC__)
|
||||
#include <vppinfra/vector_altivec.h>
|
||||
#endif
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
/*
|
||||
* Copyright (c) 2018 Cisco and/or its affiliates.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at:
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef included_vector_avx2_h
|
||||
#define included_vector_avx2_h
|
||||
|
||||
#include <vppinfra/clib.h>
|
||||
#include <x86intrin.h>
|
||||
|
||||
#define foreach_avx2_vec256i \
|
||||
_(i,8,32,epi8) _(i,16,16,epi16) _(i,32,8,epi32) _(i,64,4,epi64x)
|
||||
#define foreach_avx2_vec256u \
|
||||
_(u,8,32,epi8) _(u,16,16,epi16) _(u,32,8,epi32) _(u,64,4,epi64x)
|
||||
#define foreach_avx2_vec256f \
|
||||
_(f,32,8,ps) _(f,64,4,pd)
|
||||
|
||||
/* splat, load_unaligned, store_unaligned, is_all_zero, is_all_equal */
|
||||
#define _(t, s, c, i) \
|
||||
static_always_inline t##s##x##c \
|
||||
t##s##x##c##_splat (t##s x) \
|
||||
{ return (t##s##x##c) _mm256_set1_##i (x); } \
|
||||
\
|
||||
static_always_inline t##s##x##c \
|
||||
t##s##x##c##_load_unaligned (void *p) \
|
||||
{ return (t##s##x##c) _mm256_loadu_si256 (p); } \
|
||||
\
|
||||
static_always_inline void \
|
||||
t##s##x##c##_store_unaligned (t##s##x##c v, void *p) \
|
||||
{ _mm256_storeu_si256 ((__m256i *) p, (__m256i) v); } \
|
||||
\
|
||||
static_always_inline int \
|
||||
t##s##x##c##_is_all_zero (t##s##x##c x) \
|
||||
{ return _mm256_testz_si256 ((__m256i) x, (__m256i) x); } \
|
||||
\
|
||||
static_always_inline int \
|
||||
t##s##x##c##_is_all_equal (t##s##x##c v, t##s x) \
|
||||
{ return t##s##x##c##_is_all_zero (v != t##s##x##c##_splat (x)); }; \
|
||||
\
|
||||
|
||||
foreach_avx2_vec256i foreach_avx2_vec256u
|
||||
#undef _
|
||||
always_inline u32x8
|
||||
u32x8_permute (u32x8 v, u32x8 idx)
|
||||
{
|
||||
return (u32x8) _mm256_permutevar8x32_epi32 ((__m256i) v, (__m256i) idx);
|
||||
}
|
||||
|
||||
always_inline u32x4
|
||||
u32x8_extract_lo (u32x8 v)
|
||||
{
|
||||
return (u32x4) _mm256_extracti128_si256 ((__m256i) v, 0);
|
||||
}
|
||||
|
||||
always_inline u32x4
|
||||
u32x8_extract_hi (u32x8 v)
|
||||
{
|
||||
return (u32x4) _mm256_extracti128_si256 ((__m256i) v, 1);
|
||||
}
|
||||
|
||||
#endif /* included_vector_avx2_h */
|
||||
|
||||
/*
|
||||
* fd.io coding-style-patch-verification: ON
|
||||
*
|
||||
* Local Variables:
|
||||
* eval: (c-set-style "gnu")
|
||||
* End:
|
||||
*/
|
||||
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
* Copyright (c) 2015 Cisco and/or its affiliates.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at:
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef included_vector_avx512_h
|
||||
#define included_vector_avx512_h
|
||||
|
||||
#include <vppinfra/clib.h>
|
||||
#include <x86intrin.h>
|
||||
|
||||
#define foreach_avx512_vec512i \
|
||||
_(i,8,64,epi8) _(i,16,32,epi16) _(i,32,16,epi32) _(i,64,8,epi64)
|
||||
#define foreach_avx512_vec512u \
|
||||
_(u,8,64,epi8) _(u,16,32,epi16) _(u,32,16,epi32) _(u,64,8,epi64)
|
||||
#define foreach_avx512_vec512f \
|
||||
_(f,32,8,ps) _(f,64,4,pd)
|
||||
|
||||
/* splat, load_unaligned, store_unaligned */
|
||||
#define _(t, s, c, i) \
|
||||
static_always_inline t##s##x##c \
|
||||
t##s##x##c##_splat (t##s x) \
|
||||
{ return (t##s##x##c) _mm512_set1_##i (x); } \
|
||||
\
|
||||
static_always_inline t##s##x##c \
|
||||
t##s##x##c##_load_unaligned (void *p) \
|
||||
{ return (t##s##x##c) _mm512_loadu_si512 (p); } \
|
||||
\
|
||||
static_always_inline void \
|
||||
t##s##x##c##_store_unaligned (t##s##x##c v, void *p) \
|
||||
{ _mm512_storeu_si512 ((__m512i *) p, (__m512i) v); } \
|
||||
\
|
||||
|
||||
foreach_avx512_vec512i foreach_avx512_vec512u
|
||||
#undef _
|
||||
#endif /* included_vector_avx512_h */
|
||||
/*
|
||||
* fd.io coding-style-patch-verification: ON
|
||||
*
|
||||
* Local Variables:
|
||||
* eval: (c-set-style "gnu")
|
||||
* End:
|
||||
*/
|
||||
@@ -200,9 +200,9 @@ u64x2_write_hi (u64x2 x, u64 * a)
|
||||
/* Unaligned loads/stores. */
|
||||
|
||||
#define _(t) \
|
||||
always_inline void t##_store_unaligned (t x, t * a) \
|
||||
always_inline void t##_store_unaligned (t x, void * a) \
|
||||
{ _mm_storeu_si128 ((__m128i *) a, (__m128i) x); } \
|
||||
always_inline t t##_load_unaligned (t * a) \
|
||||
always_inline t t##_load_unaligned (void * a) \
|
||||
{ return (t) _mm_loadu_si128 ((__m128i *) a); }
|
||||
|
||||
_(u8x16) _(u16x8) _(u32x4) _(u64x2) _(i8x16) _(i16x8) _(i32x4) _(i64x2)
|
||||
|
||||
Reference in New Issue
Block a user