diff --git a/MAINTAINERS b/MAINTAINERS index 91626856663..c310e728227 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -420,6 +420,12 @@ I: ena M: Damjan Marion F: src/plugins/dev_ena/ +Plugin - Marvell Octeon device driver +I: octeon +M: Monendra Singh Kushwaha +M: Damjan Marion +F: src/plugins/dev_octeon/ + Plugin - Dispatch Trace PCAP I: dispatch-trace M: Dave Barach diff --git a/build-data/packages/vpp.mk b/build-data/packages/vpp.mk index 4dd65ecb4fc..9c8e62dc739 100644 --- a/build-data/packages/vpp.mk +++ b/build-data/packages/vpp.mk @@ -30,6 +30,9 @@ vpp_cmake_args += -DCMAKE_PREFIX_PATH:PATH="$(vpp_cmake_prefix_path)" ifeq ("$(V)","1") vpp_cmake_args += -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON endif +ifneq ($(VPP_PLATFORM),) +vpp_cmake_args += -DVPP_PLATFORM="$(VPP_PLATFORM)" +endif ifneq ($(VPP_EXCLUDED_PLUGINS),) vpp_cmake_args += -DVPP_EXCLUDED_PLUGINS="$(VPP_EXCLUDED_PLUGINS)" endif diff --git a/build/external/Makefile b/build/external/Makefile index fb3cd94e526..1e1f5da1d1f 100644 --- a/build/external/Makefile +++ b/build/external/Makefile @@ -35,6 +35,7 @@ CMAKE?=cmake endif ARCH_X86_64=$(filter x86_64,$(shell uname -m)) +AARCH64=$(filter aarch64,$(shell uname -m)) include packages.mk include packages/ipsec-mb.mk @@ -42,13 +43,14 @@ include packages/quicly.mk include packages/rdma-core.mk include packages/dpdk.mk include packages/xdp-tools.mk +include packages/octeon-roc.mk .PHONY: clean clean: @rm -rf $(B) $(I) .PHONY: install -install: $(if $(ARCH_X86_64), ipsec-mb-install) dpdk-install rdma-core-install quicly-install xdp-tools-install +install: $(if $(ARCH_X86_64), ipsec-mb-install) dpdk-install rdma-core-install quicly-install xdp-tools-install $(if $(AARCH64), octeon-roc-install) .PHONY: config config: $(if $(ARCH_X86_64), ipsec-mb-config) dpdk-config rdma-core-config quicly-build diff --git a/build/external/packages/octeon-roc.mk b/build/external/packages/octeon-roc.mk new file mode 100644 index 00000000000..4bdeabeac2d --- /dev/null +++ b/build/external/packages/octeon-roc.mk @@ -0,0 +1,26 @@ +# Copyright (c) 2023 Marvell. +# SPDX-License-Identifier: Apache-2.0 +# https://spdx.org/licenses/Apache-2.0.html + +octeon-roc_version := 0.2 +octeon-roc_tarball := octeon-roc-v$(octeon-roc_version).tar.gz +octeon-roc_tarball_md5sum := a72bb3b27fd3bbaf58aadd99514620e1 + +octeon-roc_tarball_strip_dirs := 1 +octeon-roc_url := https://github.com/MarvellEmbeddedProcessors/marvell-vpp/archive/refs/tags/$(octeon-roc_tarball) + +define octeon-roc_config_cmds + @true +endef + +define octeon-roc_build_cmds + @cd ${octeon-roc_src_dir} && rm -f $(octeon-roc_build_log) && $(CMAKE) ${octeon-roc_src_dir} -DCMAKE_INSTALL_PREFIX='$(octeon-roc_install_dir)' >> $(octeon-roc_build_log) + @$(MAKE) -C ${octeon-roc_src_dir} >> $(octeon-roc_build_log) +endef + +define octeon-roc_install_cmds + @$(MAKE) -C ${octeon-roc_src_dir} install >> $(octeon-roc_install_log) +endef + +$(eval $(call package,octeon-roc)) + diff --git a/configure b/configure index fbe27daeac3..7db7726ddc6 100755 --- a/configure +++ b/configure @@ -5,6 +5,7 @@ set -o pipefail -o errtrace -o nounset -o errexit # submitting any changes # defaults +platform=default build_dir=. install_dir=/usr/local build_type=release @@ -31,6 +32,7 @@ OPTIONS: --native-only, -n Only compile for Native CPU (no multiarch) --wipe, -w Wipe whole repo (except startup.* files) --sanitize, -s Enable sanitizer (mem) + --platform, -p Specify target platform __EOF__ } @@ -76,6 +78,15 @@ while (( "$#" )); do exit 1 fi ;; + -p|--platform) + if [ -n "$2" ] && [ ${2:0:1} != "-" ]; then + platform=$2 + shift 2 + else + echo "Error: Argument for $1 is missing" >&2 + exit 1 + fi + ;; -n|--native-only) native_only=yes shift 1 @@ -121,6 +132,7 @@ args+=("-DCMAKE_PREFIX_PATH=/opt/vpp/external/${arch}") args+=("-DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON") args+=("-DCMAKE_INSTALL_PREFIX=${install_dir}") args+=("-DCMAKE_BUILD_TYPE:STRING=${build_type}") +args+=("-DVPP_PLATFORM=${platform}") [ "${native_only}" == "yes" ] && args+=("-DVPP_BUILD_NATIVE_ONLY:BOOL=ON") [ "${wipe}" == "yes" ] && git clean -fdx --exclude=startup.\* diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 590d6cdaa02..2348c8dc391 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -249,6 +249,7 @@ det dev devbind dev_iavf +dev_octeon df dhcp dhcp @@ -786,6 +787,7 @@ O'Driscoll oacl oam OAM +octeon oddbuf Oddbuf offline diff --git a/extras/deb/mkdeb-octeon-roc b/extras/deb/mkdeb-octeon-roc new file mode 100755 index 00000000000..08e57f4bd1e --- /dev/null +++ b/extras/deb/mkdeb-octeon-roc @@ -0,0 +1,46 @@ +#!/bin/bash + +PKG=vpp-dep-octeon-roc +URL=https://github.com/MarvellEmbeddedProcessors/marvell-vpp.git +ARCH=$(dpkg --print-architecture) +TMP_DIR=$(mktemp -d -p $PWD) + +set -eEuo pipefail + +err_handler() +{ + trap '' INT TERM EXIT ERR + echo "Cleaning up ${TMP_DIR}" + rm -rf ${TMP_DIR} + exit +} +trap "err_handler" INT TERM EXIT ERR + +SRC=${TMP_DIR}/src +BUILD=${TMP_DIR}/build +STAGE=${TMP_DIR}/pkg +INSTALL_PREFIX=/opt/vpp/external/$(uname -m) + +git clone ${URL} ${SRC} +VER=0.0.$(git -C ${SRC} rev-list --count HEAD) + +cmake -S ${SRC} -B ${BUILD} +cmake --build ${BUILD} --parallel +cmake --install ${BUILD} --prefix ${STAGE}${INSTALL_PREFIX} + +mkdir -p ${STAGE}/DEBIAN + +cat > ${STAGE}/DEBIAN/control << __EOF__ +Package: ${PKG} +Version: ${VER} +Architecture: ${ARCH} +Maintainer: vpp-dev +Installed-Size: $(du -ks ${STAGE}|cut -f 1) +Section: system +Priority: extra +Description: Marvell Octeon ROC library for VPP + See https://github.com/MarvellEmbeddedProcessors/marvell-vpp +__EOF__ + +DEB=${PKG}_${VER}_${ARCH}.deb +dpkg-deb -b ${STAGE} ${DEB} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ee2882b4285..d13685b8d14 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -13,7 +13,25 @@ cmake_minimum_required(VERSION 3.13) -set(CMAKE_C_COMPILER_NAMES clang gcc cc) +if(DEFINED VPP_PLATFORM AND VPP_PLATFORM STREQUAL "default") + unset(VPP_PLATFORM) + set(VPP_PLATFORM_NAME "default") +elseif(DEFINED VPP_PLATFORM) + set(platform_file ${CMAKE_SOURCE_DIR}/cmake/platform/${VPP_PLATFORM}.cmake) + if(NOT EXISTS ${platform_file}) + message(FATAL_ERROR "unknown platform ${VPP_PLATFORM}") + endif() + include(${platform_file}) + set(VPP_PLATFORM_NAME ${VPP_PLATFORM}) +else() + set(VPP_PLATFORM_NAME "default") +endif() + +if (DEFINED VPP_PLATFORM_C_COMPILER_NAMES) + set(CMAKE_C_COMPILER_NAMES ${VPP_PLATFORM_C_COMPILER_NAME}) +else() + set(CMAKE_C_COMPILER_NAMES clang gcc cc) +endif() project(vpp C) @@ -37,6 +55,11 @@ execute_process( OUTPUT_VARIABLE VPP_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE ) + +if (VPP_PLATFORM) + set(VPP_VERSION ${VPP_VERSION}-${VPP_PLATFORM_NAME}) +endif() + string(REPLACE "-" ";" VPP_LIB_VERSION ${VPP_VERSION}) list(GET VPP_LIB_VERSION 0 VPP_LIB_VERSION) @@ -183,7 +206,7 @@ if(VPP_ENABLE_TRAJECTORY_TRACE) endif() ############################################################################## -# unittest with clang ode coverage +# unittest with clang code coverage ############################################################################## if("${CMAKE_VERSION}" VERSION_GREATER_EQUAL "3.13" AND "${CMAKE_C_COMPILER_ID}" MATCHES "(Apple)?[Cc]lang") @@ -307,6 +330,7 @@ mark_as_advanced(CLEAR # print configuration ############################################################################## message(STATUS "Configuration:") +pr("VPP platform" ${VPP_PLATFORM_NAME}) pr("VPP version" ${VPP_VERSION}) pr("VPP library version" ${VPP_LIB_VERSION}) pr("GIT toplevel dir" ${VPP_GIT_TOPLEVEL_DIR}) diff --git a/src/cmake/cpu.cmake b/src/cmake/cpu.cmake index 18a1210f541..25e966dcf7a 100644 --- a/src/cmake/cpu.cmake +++ b/src/cmake/cpu.cmake @@ -24,16 +24,21 @@ endmacro() ############################################################################## # Cache line size ############################################################################## -if(DEFINED VPP_CACHE_LINE_SIZE) - # Cache line size assigned via cmake args -elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*)") - set(VPP_CACHE_LINE_SIZE 128) -else() - set(VPP_CACHE_LINE_SIZE 64) -endif() -set(VPP_CACHE_LINE_SIZE ${VPP_CACHE_LINE_SIZE} - CACHE STRING "Target CPU cache line size") +if(DEFINED VPP_PLATFORM_CACHE_LINE_SIZE) + set(VPP_CACHE_LINE_SIZE ${VPP_PLATFORM_CACHE_LINE_SIZE}) +else() + if(DEFINED VPP_CACHE_LINE_SIZE) + # Cache line size assigned via cmake args + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*)") + set(VPP_CACHE_LINE_SIZE 128) + else() + set(VPP_CACHE_LINE_SIZE 64) + endif() + + set(VPP_CACHE_LINE_SIZE ${VPP_CACHE_LINE_SIZE} + CACHE STRING "Target CPU cache line size") +endif() set_log2_cacheline_size(VPP_LOG2_CACHE_LINE_SIZE ${VPP_CACHE_LINE_SIZE}) @@ -57,7 +62,9 @@ endif() # CPU optimizations and multiarch support ############################################################################## -option(VPP_BUILD_NATIVE_ONLY "Build only for native CPU." OFF) +if(NOT DEFINED VPP_PLATFORM) + option(VPP_BUILD_NATIVE_ONLY "Build only for native CPU." OFF) +endif() macro(add_vpp_march_variant v) cmake_parse_arguments(ARG @@ -104,14 +111,23 @@ macro(add_vpp_march_variant v) endif() endmacro() -if(VPP_BUILD_NATIVE_ONLY) +if(DEFINED VPP_PLATFORM) + if(DEFINED VPP_PLATFORM_MARCH_FLAGS) + set(VPP_DEFAULT_MARCH_FLAGS ${VPP_PLATFORM_MARCH_FLAGS}) + check_c_compiler_flag(${VPP_DEFAULT_MARCH_FLAGS} compiler_flag_march) + if(NOT compiler_flag_march) + message(FATAL_ERROR "platform build with ${VPP_DEFAULT_MARCH_FLAGS} is not supported by compiler") + endif() + else() + set(VPP_DEFAULT_MARCH_FLAGS "") + endif() + set(MARCH_VARIANTS_NAMES "platform-only") +elseif(VPP_BUILD_NATIVE_ONLY) set(VPP_BUILD_NATIVE_ARCH "native" CACHE STRING "native CPU -march= value.") set(VPP_DEFAULT_MARCH_FLAGS -march=${VPP_BUILD_NATIVE_ARCH}) - if(VPP_BUILD_NATIVE_ONLY) - check_c_compiler_flag(${VPP_DEFAULT_MARCH_FLAGS} compiler_flag_march) - if(NOT compiler_flag_march) - message(FATAL_ERROR "Native-only build with ${VPP_DEFAULT_MARCH_FLAGS} is not supported by compiler") - endif() + check_c_compiler_flag(${VPP_DEFAULT_MARCH_FLAGS} compiler_flag_march) + if(NOT compiler_flag_march) + message(FATAL_ERROR "Native-only build with ${VPP_DEFAULT_MARCH_FLAGS} is not supported by compiler") endif() set(MARCH_VARIANTS_NAMES "native-only") elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "amd64.*|x86_64.*|AMD64.*") diff --git a/src/cmake/platform/octeon10.cmake b/src/cmake/platform/octeon10.cmake new file mode 100644 index 00000000000..da7dbabdc7b --- /dev/null +++ b/src/cmake/platform/octeon10.cmake @@ -0,0 +1,3 @@ + +set(VPP_PLATFORM_CACHE_LINE_SIZE 64) +set(VPP_PLATFORM_MARCH_FLAGS -march=armv8.3-a+crypto+sve2-bitperm) diff --git a/src/plugins/dev_octeon/CMakeLists.txt b/src/plugins/dev_octeon/CMakeLists.txt new file mode 100644 index 00000000000..b7c25fe0404 --- /dev/null +++ b/src/plugins/dev_octeon/CMakeLists.txt @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright(c) 2022 Cisco Systems, Inc. + +if (NOT VPP_PLATFORM_NAME STREQUAL "octeon10") + return() +endif() + +# Find OCTEON roc files +vpp_find_path(OCTEON_ROC_DIR PATH_SUFFIXES octeon-roc NAMES platform.h) +vpp_plugin_find_library(dev-octeon OCTEON_ROC_LIB "libocteon-roc.a") + +if (NOT OCTEON_ROC_DIR) + message("OCTEON ROC files not found - Marvell OCTEON device plugin disabled") + return() +endif() + +if (NOT OCTEON_ROC_LIB) + message("OCTEON ROC library (libocteon-roc.a) not found - Marvell OCTEON device plugin disabled") + return () +endif() + +include_directories (${OCTEON_ROC_DIR}/) + +add_vpp_plugin(dev_octeon + SOURCES + init.c + format.c + port.c + queue.c + roc_helper.c + rx_node.c + tx_node.c + + MULTIARCH_SOURCES + rx_node.c + tx_node.c + + LINK_LIBRARIES + ${OCTEON_ROC_LIB} +) + diff --git a/src/plugins/dev_octeon/common.h b/src/plugins/dev_octeon/common.h new file mode 100644 index 00000000000..a7a051526d2 --- /dev/null +++ b/src/plugins/dev_octeon/common.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2023 Cisco Systems, Inc. + */ + +#ifndef _OCT_COMMON_H_ +#define _OCT_COMMON_H_ + +#include +#include +#include +#include +#include + +static_always_inline u32 +oct_aura_free_all_buffers (vlib_main_t *vm, u64 aura_handle, u16 hdr_off) +{ + u32 n = 0; + u64 iova; + + while ((iova = roc_npa_aura_op_alloc (aura_handle, 0))) + { + vlib_buffer_t *b = (void *) iova + hdr_off; + vlib_buffer_free_one (vm, vlib_get_buffer_index (vm, b)); + n++; + } + return n; +} + +#endif /* _OCT_COMMON_H_ */ diff --git a/src/plugins/dev_octeon/format.c b/src/plugins/dev_octeon/format.c new file mode 100644 index 00000000000..5ee956ad4f4 --- /dev/null +++ b/src/plugins/dev_octeon/format.c @@ -0,0 +1,164 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2023 Cisco Systems, Inc. + */ + +#include "vlib/pci/pci.h" +#include "vnet/error.h" +#include "vppinfra/error.h" +#include +#include +#include + +u8 * +format_oct_port_status (u8 *s, va_list *args) +{ + return s; +} + +u8 * +format_oct_nix_rx_cqe_desc (u8 *s, va_list *args) +{ + oct_nix_rx_cqe_desc_t *d = va_arg (*args, oct_nix_rx_cqe_desc_t *); + u32 indent = format_get_indent (s); + typeof (d->hdr) *h = &d->hdr; + typeof (d->parse.f) *p = &d->parse.f; + typeof (d->sg0) *sg0 = &d->sg0; + typeof (d->sg0) *sg1 = &d->sg1; + + s = format (s, "hdr: cqe_type %u nude %u q %u tag 0x%x", h->cqe_type, + h->node, h->q, h->tag); + s = format (s, "\n%Uparse:", format_white_space, indent); +#define _(n, f) s = format (s, " " #n " " f, p->n) + _ (chan, "%u"); + _ (errcode, "%u"); + _ (errlev, "%u"); + _ (desc_sizem1, "%u"); + _ (pkt_lenm1, "%u"); + _ (pkind, "%u"); + s = format (s, "\n%U ", format_white_space, indent); + _ (nix_idx, "%u"); + _ (color, "%u"); + _ (flow_key_alg, "%u"); + _ (eoh_ptr, "%u"); + _ (match_id, "0x%x"); + s = format (s, "\n%U ", format_white_space, indent); + _ (wqe_aura, "0x%x"); + _ (pb_aura, "0x%x"); + _ (imm_copy, "%u"); + _ (express, "%u"); + _ (wqwd, "%u"); + _ (l2m, "%u"); + _ (l2b, "%u"); + _ (l3m, "%u"); + _ (l3b, "%u"); +#undef _ + s = format (s, "\n%U ", format_white_space, indent); + s = format (s, "layer: a b c d e f g h"); + s = format (s, "\n%U ", format_white_space, indent); + s = format (s, "type: %3u %3u %3u %3u %3u %3u %3u %3u", p->latype, + p->lbtype, p->lctype, p->ldtype, p->letype, p->lftype, p->lgtype, + p->lhtype); + s = format (s, "\n%U ", format_white_space, indent); + s = format ( + s, "flags: 0x%02x 0x%02x 0x%02x 0x%02x 0x%02x 0x%02x 0x%02x 0x%02x", + p->laflags, p->lbflags, p->lcflags, p->ldflags, p->leflags, p->lfflags, + p->lgflags, p->lhflags); + s = format (s, "\n%U ", format_white_space, indent); + s = format (s, "ptr: %3u %3u %3u %3u %3u %3u %3u %3u", p->laptr, + p->lbptr, p->lcptr, p->ldptr, p->leptr, p->lfptr, p->lgptr, + p->lhptr); + + if (sg0->subdc != 0x4) + return format (s, "\n%Usg0: unexpected subdc %x", format_white_space, + indent, sg0->subdc); + + s = format (s, + "\n%Usg0: segs %u seg1_sz %u seg2_sz %u seg3_sz %u seg1 " + "%p seg2 %p seg3 %p", + format_white_space, indent, sg0->segs, sg0->seg1_size, + sg0->seg2_size, sg0->seg3_size, d->segs0[0], d->segs0[1], + d->segs0[2]); + + if (sg1->subdc != 0x4 && sg1->subdc != 0) + return format (s, "\n%Usg1: unexpected subdc %x", format_white_space, + indent, sg1->subdc); + + if (sg1->subdc == 4) + s = format (s, + "\n%Usg1: segs %u seg1_sz %u seg2_sz %u seg3_sz %u seg1 " + "%p seg2 %p seg3 %p", + format_white_space, indent, sg1->segs, sg1->seg1_size, + sg1->seg2_size, sg1->seg3_size, d->segs1[0], d->segs1[1], + d->segs1[2]); + + return s; +} + +u8 * +format_oct_rx_trace (u8 *s, va_list *args) +{ + vlib_main_t *vm = va_arg (*args, vlib_main_t *); + vlib_node_t *node = va_arg (*args, vlib_node_t *); + oct_rx_trace_t *t = va_arg (*args, oct_rx_trace_t *); + u32 indent = format_get_indent (s); + + s = format (s, "octeon-rx: next-node %U sw_if_index %u", + format_vlib_next_node_name, vm, node->index, t->next_index, + t->sw_if_index); + s = format (s, "\n%U%U", format_white_space, indent + 2, + format_oct_nix_rx_cqe_desc, &t->desc); + return s; +} + +u8 * +format_oct_tx_trace (u8 *s, va_list *args) +{ + va_arg (*args, vlib_main_t *); + va_arg (*args, vlib_node_t *); + oct_tx_trace_t *t = va_arg (*args, oct_tx_trace_t *); + u32 indent = format_get_indent (s); + + s = format (s, "octeon-tx: sw_if_index %u", t->sw_if_index); + s = format (s, "\n%Uhdr[0]:", format_white_space, indent + 2); +#define _(n, f) s = format (s, " " #n " " f, t->desc.hdr_w0.n) + _ (total, "%u"); + _ (df, "%u"); + _ (aura, "0x%x"); + _ (sizem1, "%u"); + _ (pnc, "%u"); + _ (sq, "%u"); +#undef _ + s = format (s, "\n%Uhdr[1]:", format_white_space, indent + 2); +#define _(n, f) s = format (s, " " #n " " f, t->desc.hdr_w1.n) + _ (ol3ptr, "%u"); + _ (ol4ptr, "%u"); + _ (il3ptr, "%u"); + _ (il4ptr, "%u"); + _ (ol3type, "%u"); + _ (ol4type, "%u"); + _ (il3type, "%u"); + _ (il4type, "%u"); + _ (sqe_id, "%u"); +#undef _ + + foreach_int (j, 0, 4) + { + s = format (s, "\n%Usg[%u]:", format_white_space, indent + 2, j); +#define _(n, f) s = format (s, " " #n " " f, t->desc.sg[j].n) + _ (subdc, "%u"); + _ (segs, "%u"); + _ (seg1_size, "%u"); + _ (seg2_size, "%u"); + _ (seg3_size, "%u"); + _ (i1, "%u"); + _ (i2, "%u"); + _ (i3, "%u"); + _ (ld_type, "%u"); +#undef _ + for (int i = 1; i < 4; i++) + s = format (s, "\n%Usg[%u]: %p", format_white_space, indent + 2, i + j, + t->desc.sg[i + j]); + } + + return s; +} diff --git a/src/plugins/dev_octeon/hw_defs.h b/src/plugins/dev_octeon/hw_defs.h new file mode 100644 index 00000000000..ab0fc7bd8da --- /dev/null +++ b/src/plugins/dev_octeon/hw_defs.h @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2023 Cisco Systems, Inc. + */ + +#ifndef _OCT_HW_DEFS_H_ +#define _OCT_HW_DEFS_H_ + +#include +#include + +typedef union +{ + struct + { + u64 tail : 20; + u64 head : 20; + u64 resv40 : 6; + u64 cq_err : 1; + u64 resv47 : 16; + u64 op_err : 1; + }; + u64 as_u64; +} oct_nix_lf_cq_op_status_t; + +STATIC_ASSERT_SIZEOF (oct_nix_lf_cq_op_status_t, 8); + +typedef union +{ + struct + { + u64 aura : 20; + u64 _reseved20 : 12; + u64 count_eot : 1; + u64 _reserved33 : 30; + u64 fabs : 1; + }; + u64 as_u64; +} oct_npa_lf_aura_batch_free0_t; + +STATIC_ASSERT_SIZEOF (oct_npa_lf_aura_batch_free0_t, 8); + +typedef struct +{ + oct_npa_lf_aura_batch_free0_t w0; + u64 data[15]; +} oct_npa_lf_aura_batch_free_line_t; + +STATIC_ASSERT_SIZEOF (oct_npa_lf_aura_batch_free_line_t, 128); + +typedef union +{ + struct npa_batch_alloc_compare_s compare_s; + u64 as_u64; +} oct_npa_batch_alloc_compare_t; + +typedef union +{ + struct + { + union nix_send_hdr_w0_u hdr_w0; + union nix_send_hdr_w1_u hdr_w1; + union nix_send_sg_s sg[8]; + }; + u128 as_u128[5]; +} oct_tx_desc_t; + +STATIC_ASSERT_SIZEOF (oct_tx_desc_t, 80); + +typedef union +{ + u128 dwords[8]; + u64 words[16]; +} lmt_line_t; + +STATIC_ASSERT_SIZEOF (lmt_line_t, 1 << ROC_LMT_LINE_SIZE_LOG2); + +typedef union +{ + union nix_rx_parse_u f; + u64 w[7]; +} oct_nix_rx_parse_t; + +STATIC_ASSERT_SIZEOF (oct_nix_rx_parse_t, 56); + +typedef struct +{ + CLIB_ALIGN_MARK (desc, 128); + struct nix_cqe_hdr_s hdr; + oct_nix_rx_parse_t parse; + struct nix_rx_sg_s sg0; + void *segs0[3]; + struct nix_rx_sg_s sg1; + void *segs1[3]; +} oct_nix_rx_cqe_desc_t; + +STATIC_ASSERT_SIZEOF (oct_nix_rx_cqe_desc_t, 128); + +#endif /* _OCT_HW_DEFS_H_ */ diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c new file mode 100644 index 00000000000..bee449f2123 --- /dev/null +++ b/src/plugins/dev_octeon/init.c @@ -0,0 +1,301 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2023 Cisco Systems, Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +struct roc_model oct_model; + +VLIB_REGISTER_LOG_CLASS (oct_log, static) = { + .class_name = "octeon", + .subclass_name = "init", +}; + +#define _(f, n, s, d) \ + { .name = #n, .desc = d, .severity = VL_COUNTER_SEVERITY_##s }, + +vlib_error_desc_t oct_tx_node_counters[] = { foreach_oct_tx_node_counter }; +#undef _ + +vnet_dev_node_t oct_rx_node = { + .format_trace = format_oct_rx_trace, +}; + +vnet_dev_node_t oct_tx_node = { + .format_trace = format_oct_tx_trace, + .error_counters = oct_tx_node_counters, + .n_error_counters = ARRAY_LEN (oct_tx_node_counters), +}; + +static struct +{ + u16 device_id; + oct_device_type_t type; + char *description; +} oct_dev_types[] = { + +#define _(id, device_type, desc) \ + { \ + .device_id = (id), .type = OCT_DEVICE_TYPE_##device_type, \ + .description = (desc) \ + } + + _ (0xa063, RVU_PF, "Marvell Octeon Resource Virtualization Unit PF"), + _ (0xa0f3, CPT_VF, "Marvell Octeon Cryptographic Accelerator Unit VF"), +#undef _ +}; + +static u8 * +oct_probe (vlib_main_t *vm, vnet_dev_bus_index_t bus_index, void *dev_info) +{ + vnet_dev_bus_pci_device_info_t *di = dev_info; + + if (di->vendor_id != 0x177d) /* Cavium */ + return 0; + + FOREACH_ARRAY_ELT (dt, oct_dev_types) + { + if (dt->device_id == di->device_id) + return format (0, "%s", dt->description); + } + + return 0; +} + +vnet_dev_rv_t +cnx_return_roc_err (vnet_dev_t *dev, int rrv, char *fmt, ...) +{ + va_list va; + va_start (va, fmt); + u8 *s = va_format (0, fmt, &va); + va_end (va); + + log_err (dev, "%v: %s [%d]", s, roc_error_msg_get (rrv), rrv); + vec_free (s); + + return VNET_DEV_ERR_UNSUPPORTED_DEVICE; +} + +static vnet_dev_rv_t +oct_alloc (vlib_main_t *vm, vnet_dev_t *dev) +{ + oct_device_t *cd = vnet_dev_get_data (dev); + cd->nix = + clib_mem_alloc_aligned (sizeof (struct roc_nix), CLIB_CACHE_LINE_BYTES); + return VNET_DEV_OK; +} + +static vnet_dev_rv_t +oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) +{ + oct_device_t *cd = vnet_dev_get_data (dev); + u8 mac_addr[6]; + int rrv; + oct_port_t oct_port = {}; + + *cd->nix = (struct roc_nix){ + .reta_sz = ROC_NIX_RSS_RETA_SZ_256, + .max_sqb_count = 512, + .pci_dev = &cd->plt_pci_dev, + }; + + if ((rrv = roc_nix_dev_init (cd->nix))) + return cnx_return_roc_err (dev, rrv, "roc_nix_dev_init"); + + if (roc_nix_npc_mac_addr_get (cd->nix, mac_addr)) + return cnx_return_roc_err (dev, rrv, "roc_nix_npc_mac_addr_get"); + + vnet_dev_port_add_args_t port_add_args = { + .port = { + .attr = { + .type = VNET_DEV_PORT_TYPE_ETHERNET, + .max_rx_queues = 64, + .max_tx_queues = 64, + .max_supported_rx_frame_size = roc_nix_max_pkt_len (cd->nix), + .caps = { + .rss = 1, + }, + .rx_offloads = { + .ip4_cksum = 1, + }, + }, + .ops = { + .init = oct_port_init, + .deinit = oct_port_deinit, + .start = oct_port_start, + .stop = oct_port_stop, + .config_change = oct_port_cfg_change, + .format_status = format_oct_port_status, + }, + .data_size = sizeof (oct_port_t), + .initial_data = &oct_port, + }, + .rx_node = &oct_rx_node, + .tx_node = &oct_tx_node, + .rx_queue = { + .config = { + .data_size = sizeof (oct_rxq_t), + .default_size = 1024, + .multiplier = 32, + .min_size = 256, + .max_size = 16384, + }, + .ops = { + .alloc = oct_rx_queue_alloc, + .free = oct_rx_queue_free, + .format_info = format_oct_rxq_info, + }, + }, + .tx_queue = { + .config = { + .data_size = sizeof (oct_txq_t), + .default_size = 1024, + .multiplier = 32, + .min_size = 256, + .max_size = 16384, + }, + .ops = { + .alloc = oct_tx_queue_alloc, + .free = oct_tx_queue_free, + .format_info = format_oct_txq_info, + }, + }, + }; + + vnet_dev_set_hw_addr_eth_mac (&port_add_args.port.attr.hw_addr, mac_addr); + + log_info (dev, "MAC address is %U", format_ethernet_address, mac_addr); + + return vnet_dev_port_add (vm, dev, 0, &port_add_args); +} + +static vnet_dev_rv_t +oct_init_cpt (vlib_main_t *vm, vnet_dev_t *dev) +{ + oct_device_t *cd = vnet_dev_get_data (dev); + int rrv; + struct roc_cpt cpt = { + .pci_dev = &cd->plt_pci_dev, + }; + + if ((rrv = roc_cpt_dev_init (&cpt))) + return cnx_return_roc_err (dev, rrv, "roc_cpt_dev_init"); + return VNET_DEV_OK; +} + +static vnet_dev_rv_t +oct_init (vlib_main_t *vm, vnet_dev_t *dev) +{ + oct_device_t *cd = vnet_dev_get_data (dev); + vlib_pci_config_hdr_t pci_hdr; + vnet_dev_rv_t rv; + + rv = vnet_dev_pci_read_config_header (vm, dev, &pci_hdr); + if (rv != VNET_DEV_OK) + return rv; + + if (pci_hdr.vendor_id != 0x177d) + return VNET_DEV_ERR_UNSUPPORTED_DEVICE; + + FOREACH_ARRAY_ELT (dt, oct_dev_types) + { + if (dt->device_id == pci_hdr.device_id) + cd->type = dt->type; + } + + if (cd->type == OCT_DEVICE_TYPE_UNKNOWN) + return rv; + + rv = VNET_DEV_ERR_UNSUPPORTED_DEVICE; + + cd->plt_pci_dev = (struct plt_pci_device){ + .id.vendor_id = pci_hdr.vendor_id, + .id.device_id = pci_hdr.device_id, + .id.class_id = pci_hdr.class << 16 | pci_hdr.subclass, + .pci_handle = vnet_dev_get_pci_handle (dev), + }; + + foreach_int (i, 2, 4) + { + rv = vnet_dev_pci_map_region (vm, dev, i, + &cd->plt_pci_dev.mem_resource[i].addr); + if (rv != VNET_DEV_OK) + return rv; + } + + strncpy ((char *) cd->plt_pci_dev.name, dev->device_id, + sizeof (cd->plt_pci_dev.name) - 1); + + if (cd->type == OCT_DEVICE_TYPE_RVU_PF) + return oct_init_nix (vm, dev); + else if (cd->type == OCT_DEVICE_TYPE_CPT_VF) + return oct_init_cpt (vm, dev); + else + return VNET_DEV_ERR_UNSUPPORTED_DEVICE; + + return 0; +} + +static void +oct_deinit (vlib_main_t *vm, vnet_dev_t *dev) +{ + oct_device_t *cd = vnet_dev_get_data (dev); + + if (cd->nix_initialized) + roc_nix_dev_fini (cd->nix); +} + +static void +oct_free (vlib_main_t *vm, vnet_dev_t *dev) +{ + oct_device_t *cd = vnet_dev_get_data (dev); + + if (cd->nix_initialized) + roc_nix_dev_fini (cd->nix); +} + +VNET_DEV_REGISTER_DRIVER (octeon) = { + .name = "octeon", + .bus = "pci", + .device_data_sz = sizeof (oct_device_t), + .ops = { + .alloc = oct_alloc, + .init = oct_init, + .deinit = oct_deinit, + .free = oct_free, + .probe = oct_probe, + }, +}; + +static clib_error_t * +oct_plugin_init (vlib_main_t *vm) +{ + int rv; + extern oct_plt_init_param_t oct_plt_init_param; + + rv = oct_plt_init (&oct_plt_init_param); + if (rv) + return clib_error_return (0, "oct_plt_init failed"); + + rv = roc_model_init (&oct_model); + if (rv) + return clib_error_return (0, "roc_model_init failed"); + return 0; +} + +VLIB_INIT_FUNCTION (oct_plugin_init); + +VLIB_PLUGIN_REGISTER () = { + .version = VPP_BUILD_VER, + .description = "dev_octeon", +}; diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h new file mode 100644 index 00000000000..a87330c6f90 --- /dev/null +++ b/src/plugins/dev_octeon/octeon.h @@ -0,0 +1,154 @@ + +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2023 Cisco Systems, Inc. + */ +#ifndef _OCTEON_H_ +#define _OCTEON_H_ +#include +#include +#include +#include +#include +#include +#include + +typedef enum +{ + OCT_DEVICE_TYPE_UNKNOWN = 0, + OCT_DEVICE_TYPE_RVU_PF, + OCT_DEVICE_TYPE_CPT_VF, +} __clib_packed oct_device_type_t; + +typedef struct +{ + oct_device_type_t type; + u8 nix_initialized : 1; + u8 status : 1; + u8 full_duplex : 1; + u32 speed; + struct plt_pci_device plt_pci_dev; + struct roc_cpt cpt; + struct roc_nix *nix; +} oct_device_t; + +typedef struct +{ + u8 lf_allocated : 1; + u8 tm_initialized : 1; + u8 npc_initialized : 1; + struct roc_npc npc; +} oct_port_t; + +typedef struct +{ + u8 npa_pool_initialized : 1; + u8 cq_initialized : 1; + u8 rq_initialized : 1; + u16 hdr_off; + u32 n_enq; + u64 aura_handle; + u64 aura_batch_free_ioaddr; + u64 lmt_base_addr; + CLIB_CACHE_LINE_ALIGN_MARK (data0); + struct roc_nix_cq cq; + struct roc_nix_rq rq; +} oct_rxq_t; + +typedef struct +{ + CLIB_ALIGN_MARK (cl, 128); + union + { + struct npa_batch_alloc_status_s status; + u64 iova[16]; + }; +} oct_npa_batch_alloc_cl128_t; + +STATIC_ASSERT_SIZEOF (oct_npa_batch_alloc_cl128_t, 128); + +typedef struct +{ + u8 sq_initialized : 1; + u8 npa_pool_initialized : 1; + u16 hdr_off; + u32 n_enq; + u64 aura_handle; + u64 io_addr; + void *lmt_addr; + + oct_npa_batch_alloc_cl128_t *ba_buffer; + u8 ba_first_cl; + u8 ba_num_cl; + CLIB_CACHE_LINE_ALIGN_MARK (data0); + struct roc_nix_sq sq; +} oct_txq_t; + +/* format.c */ +format_function_t format_oct_port_status; +format_function_t format_oct_rx_trace; +format_function_t format_oct_tx_trace; + +/* port.c */ +vnet_dev_rv_t oct_port_init (vlib_main_t *, vnet_dev_port_t *); +vnet_dev_rv_t oct_port_start (vlib_main_t *, vnet_dev_port_t *); +void oct_port_stop (vlib_main_t *, vnet_dev_port_t *); +void oct_port_deinit (vlib_main_t *, vnet_dev_port_t *); +vnet_dev_rv_t oct_port_cfg_change (vlib_main_t *, vnet_dev_port_t *, + vnet_dev_port_cfg_change_req_t *); + +/* queue.c */ +vnet_dev_rv_t oct_rx_queue_alloc (vlib_main_t *, vnet_dev_rx_queue_t *); +vnet_dev_rv_t oct_tx_queue_alloc (vlib_main_t *, vnet_dev_tx_queue_t *); +void oct_rx_queue_free (vlib_main_t *, vnet_dev_rx_queue_t *); +void oct_tx_queue_free (vlib_main_t *, vnet_dev_tx_queue_t *); +vnet_dev_rv_t oct_rxq_init (vlib_main_t *, vnet_dev_rx_queue_t *); +vnet_dev_rv_t oct_txq_init (vlib_main_t *, vnet_dev_tx_queue_t *); +void oct_rxq_deinit (vlib_main_t *, vnet_dev_rx_queue_t *); +void oct_txq_deinit (vlib_main_t *, vnet_dev_tx_queue_t *); +format_function_t format_oct_rxq_info; +format_function_t format_oct_txq_info; + +#define log_debug(dev, f, ...) \ + vlib_log (VLIB_LOG_LEVEL_DEBUG, oct_log.class, "%U: " f, \ + format_vnet_dev_addr, (dev), ##__VA_ARGS__) +#define log_info(dev, f, ...) \ + vlib_log (VLIB_LOG_LEVEL_INFO, oct_log.class, "%U: " f, \ + format_vnet_dev_addr, (dev), ##__VA_ARGS__) +#define log_notice(dev, f, ...) \ + vlib_log (VLIB_LOG_LEVEL_NOTICE, oct_log.class, "%U: " f, \ + format_vnet_dev_addr, (dev), ##__VA_ARGS__) +#define log_warn(dev, f, ...) \ + vlib_log (VLIB_LOG_LEVEL_WARNING, oct_log.class, "%U: " f, \ + format_vnet_dev_addr, (dev), ##__VA_ARGS__) +#define log_err(dev, f, ...) \ + vlib_log (VLIB_LOG_LEVEL_ERR, oct_log.class, "%U: " f, \ + format_vnet_dev_addr, (dev), ##__VA_ARGS__) + +#define foreach_oct_tx_node_counter \ + _ (CHAIN_TOO_LONG, chain_too_long, ERROR, "drop due to buffer chain > 6") \ + _ (NO_FREE_SLOTS, no_free_slots, ERROR, "no free tx slots") \ + _ (AURA_BATCH_ALLOC_ISSUE_FAIL, aura_batch_alloc_issue_fail, ERROR, \ + "aura batch alloc issue failed") \ + _ (AURA_BATCH_ALLOC_NOT_READY, aura_batch_alloc_not_ready, ERROR, \ + "aura batch alloc not ready") + +typedef enum +{ +#define _(f, n, s, d) OCT_TX_NODE_CTR_##f, + foreach_oct_tx_node_counter +#undef _ +} oct_tx_node_counter_t; + +typedef struct +{ + u32 sw_if_index; + u32 next_index; + oct_nix_rx_cqe_desc_t desc; +} oct_rx_trace_t; + +typedef struct +{ + u32 sw_if_index; + oct_tx_desc_t desc; +} oct_tx_trace_t; +#endif /* _OCTEON_H_ */ diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c new file mode 100644 index 00000000000..5857bc15f77 --- /dev/null +++ b/src/plugins/dev_octeon/port.c @@ -0,0 +1,418 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2023 Cisco Systems, Inc. + */ + +#include +#include +#include +#include +#include +#include +#include + +VLIB_REGISTER_LOG_CLASS (oct_log, static) = { + .class_name = "octeon", + .subclass_name = "port", +}; + +static const u8 default_rss_key[] = { + 0xfe, 0xed, 0x0b, 0xad, 0xfe, 0xed, 0x0b, 0xad, 0xad, 0x0b, 0xed, 0xfe, + 0xad, 0x0b, 0xed, 0xfe, 0x13, 0x57, 0x9b, 0xef, 0x24, 0x68, 0xac, 0x0e, + 0x91, 0x72, 0x53, 0x11, 0x82, 0x64, 0x20, 0x44, 0x12, 0xef, 0x34, 0xcd, + 0x56, 0xbc, 0x78, 0x9a, 0x9a, 0x78, 0xbc, 0x56, 0xcd, 0x34, 0xef, 0x12 +}; + +static const u32 default_rss_flowkey = + (FLOW_KEY_TYPE_IPV4 | FLOW_KEY_TYPE_IPV6 | FLOW_KEY_TYPE_TCP | + FLOW_KEY_TYPE_UDP | FLOW_KEY_TYPE_SCTP); + +static const u64 rxq_cfg = + ROC_NIX_LF_RX_CFG_DIS_APAD | ROC_NIX_LF_RX_CFG_IP6_UDP_OPT | + ROC_NIX_LF_RX_CFG_L2_LEN_ERR | ROC_NIX_LF_RX_CFG_DROP_RE | + ROC_NIX_LF_RX_CFG_CSUM_OL4 | ROC_NIX_LF_RX_CFG_CSUM_IL4 | + ROC_NIX_LF_RX_CFG_LEN_OL3 | ROC_NIX_LF_RX_CFG_LEN_OL4 | + ROC_NIX_LF_RX_CFG_LEN_IL3 | ROC_NIX_LF_RX_CFG_LEN_IL4; + +static vnet_dev_rv_t +oct_roc_err (vnet_dev_t *dev, int rv, char *fmt, ...) +{ + u8 *s = 0; + va_list va; + + va_start (va, fmt); + s = va_format (s, fmt, &va); + va_end (va); + + log_err (dev, "%v - ROC error %s (%d)", s, roc_error_msg_get (rv), rv); + + vec_free (s); + return VNET_DEV_ERR_INTERNAL; +} + +vnet_dev_rv_t +oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) +{ + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + oct_port_t *cp = vnet_dev_get_port_data (port); + struct roc_nix *nix = cd->nix; + vnet_dev_rv_t rv; + int rrv; + + log_debug (dev, "port init: port %u", port->port_id); + + if ((rrv = roc_nix_lf_alloc (nix, port->intf.num_rx_queues, + port->intf.num_tx_queues, rxq_cfg))) + { + oct_port_deinit (vm, port); + return oct_roc_err ( + dev, rrv, + "roc_nix_lf_alloc(nb_rxq = %u, nb_txq = %d, rxq_cfg=0x%lx) failed", + port->intf.num_rx_queues, port->intf.num_tx_queues, rxq_cfg); + } + cp->lf_allocated = 1; + + if ((rrv = roc_nix_tm_init (nix))) + { + oct_port_deinit (vm, port); + return oct_roc_err (dev, rrv, "roc_nix_tm_init() failed"); + } + cp->tm_initialized = 1; + + if ((rrv = roc_nix_tm_hierarchy_enable (nix, ROC_NIX_TM_DEFAULT, + /* xmit_enable */ 0))) + { + oct_port_deinit (vm, port); + return oct_roc_err (dev, rrv, "roc_nix_tm_hierarchy_enable() failed"); + } + + if ((rrv = roc_nix_rss_default_setup (nix, default_rss_flowkey))) + { + oct_port_deinit (vm, port); + return oct_roc_err (dev, rrv, "roc_nix_rss_default_setup() failed"); + } + + roc_nix_rss_key_set (nix, default_rss_key); + + cp->npc.roc_nix = nix; + if ((rrv = roc_npc_init (&cp->npc))) + { + oct_port_deinit (vm, port); + return oct_roc_err (dev, rrv, "roc_npc_init() failed"); + } + cp->npc_initialized = 1; + + foreach_vnet_dev_port_rx_queue (q, port) + if (q->enabled) + if ((rv = oct_rxq_init (vm, q))) + { + oct_port_deinit (vm, port); + return rv; + } + + foreach_vnet_dev_port_tx_queue (q, port) + if (q->enabled) + if ((rv = oct_txq_init (vm, q))) + { + oct_port_deinit (vm, port); + return rv; + } + + return VNET_DEV_OK; +} + +void +oct_port_deinit (vlib_main_t *vm, vnet_dev_port_t *port) +{ + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + oct_port_t *cp = vnet_dev_get_port_data (port); + struct roc_nix *nix = cd->nix; + int rrv; + + foreach_vnet_dev_port_rx_queue (q, port) + oct_rxq_deinit (vm, q); + foreach_vnet_dev_port_tx_queue (q, port) + oct_txq_deinit (vm, q); + + if (cp->npc_initialized) + { + if ((rrv = roc_npc_fini (&cp->npc))) + oct_roc_err (dev, rrv, "roc_npc_fini() failed"); + cp->npc_initialized = 0; + } + + if (cp->tm_initialized) + { + roc_nix_tm_fini (nix); + cp->tm_initialized = 0; + } + + if (cp->lf_allocated) + { + if ((rrv = roc_nix_lf_free (nix))) + oct_roc_err (dev, rrv, "roc_nix_lf_free() failed"); + cp->lf_allocated = 0; + } +} + +void +oct_port_poll (vlib_main_t *vm, vnet_dev_port_t *port) +{ + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + struct roc_nix_link_info link_info = {}; + vnet_dev_port_state_changes_t changes = {}; + int rrv; + + rrv = roc_nix_mac_link_info_get (nix, &link_info); + if (rrv) + return; + + if (cd->status != link_info.status) + { + changes.change.link_state = 1; + changes.link_state = link_info.status; + cd->status = link_info.status; + } + + if (cd->full_duplex != link_info.full_duplex) + { + changes.change.link_duplex = 1; + changes.full_duplex = link_info.full_duplex; + cd->full_duplex = link_info.full_duplex; + } + + if (cd->speed != link_info.speed) + { + changes.change.link_speed = 1; + changes.link_speed = link_info.speed; + cd->speed = link_info.speed; + } + + if (changes.change.any == 0) + return; + + log_debug (dev, + "status %u full_duplex %u speed %u port %u lmac_type_id %u " + "fec %u aautoneg %u", + link_info.status, link_info.full_duplex, link_info.speed, + link_info.port, link_info.lmac_type_id, link_info.fec, + link_info.autoneg); + vnet_dev_port_state_change (vm, port, changes); +} + +vnet_dev_rv_t +oct_rxq_start (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq) +{ + vnet_dev_t *dev = rxq->port->dev; + oct_rxq_t *crq = vnet_dev_get_rx_queue_data (rxq); + u32 buffer_indices[rxq->size], n_alloc; + u8 bpi = vnet_dev_get_rx_queue_buffer_pool_index (rxq); + int rrv; + + n_alloc = vlib_buffer_alloc_from_pool (vm, buffer_indices, rxq->size, bpi); + + for (int i = 0; i < n_alloc; i++) + roc_npa_aura_op_free ( + crq->aura_handle, 0, + pointer_to_uword (vlib_get_buffer (vm, buffer_indices[i])) - + crq->hdr_off); + + crq->n_enq = n_alloc; + + if (roc_npa_aura_op_available (crq->aura_handle) != rxq->size) + log_warn (rxq->port->dev, "rx queue %u aura not filled completelly", + rxq->queue_id); + + if ((rrv = roc_nix_rq_ena_dis (&crq->rq, 1))) + return oct_roc_err (dev, rrv, "roc_nix_rq_ena_dis() failed"); + + return VNET_DEV_OK; +} +void +oct_rxq_stop (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq) +{ + vnet_dev_t *dev = rxq->port->dev; + oct_rxq_t *crq = vnet_dev_get_rx_queue_data (rxq); + int rrv; + u32 n; + + if ((rrv = roc_nix_rq_ena_dis (&crq->rq, 0))) + oct_roc_err (dev, rrv, "roc_nix_rq_ena_dis() failed"); + + n = oct_aura_free_all_buffers (vm, crq->aura_handle, crq->hdr_off); + + if (crq->n_enq - n > 0) + log_err (dev, "%u buffers leaked on rx queue %u stop", crq->n_enq - n, + rxq->queue_id); + else + log_debug (dev, "%u buffers freed from rx queue %u", n, rxq->queue_id); + + crq->n_enq = 0; +} + +void +oct_txq_stop (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) +{ + vnet_dev_t *dev = txq->port->dev; + oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); + oct_npa_batch_alloc_cl128_t *cl; + u32 n, off = ctq->hdr_off; + + n = oct_aura_free_all_buffers (vm, ctq->aura_handle, off); + ctq->n_enq -= n; + + if (ctq->n_enq > 0 && ctq->ba_num_cl > 0) + for (n = ctq->ba_num_cl, cl = ctq->ba_buffer + ctq->ba_first_cl; n; + cl++, n--) + { + if (cl->status.ccode != 0) + for (u32 i = 0; i < cl->status.count; i++) + { + vlib_buffer_t *b = (vlib_buffer_t *) (cl->iova[i] + off); + vlib_buffer_free_one (vm, vlib_get_buffer_index (vm, b)); + ctq->n_enq--; + } + } + + if (ctq->n_enq > 0) + log_err (dev, "%u buffers leaked on tx queue %u stop", ctq->n_enq, + txq->queue_id); + else + log_debug (dev, "%u buffers freed from tx queue %u", n, txq->queue_id); + + ctq->n_enq = 0; +} + +vnet_dev_rv_t +oct_port_start (vlib_main_t *vm, vnet_dev_port_t *port) +{ + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + struct roc_nix_eeprom_info eeprom_info = {}; + vnet_dev_rv_t rv; + int rrv; + + log_debug (port->dev, "port start: port %u", port->port_id); + + foreach_vnet_dev_port_rx_queue (q, port) + if ((rv = oct_rxq_start (vm, q)) != VNET_DEV_OK) + goto done; + + foreach_vnet_dev_port_tx_queue (q, port) + { + oct_txq_t *ctq = vnet_dev_get_tx_queue_data (q); + ctq->n_enq = 0; + } + + if ((rrv = roc_nix_mac_mtu_set (nix, 9200))) + { + rv = oct_roc_err (dev, rrv, "roc_nix_mac_mtu_set() failed"); + goto done; + } + + if ((rrv = roc_nix_npc_rx_ena_dis (nix, true))) + { + rv = oct_roc_err (dev, rrv, "roc_nix_npc_rx_ena_dis() failed"); + goto done; + } + + vnet_dev_poll_port_add (vm, port, 0.5, oct_port_poll); + + if (roc_nix_eeprom_info_get (nix, &eeprom_info) == 0) + { + log_debug (dev, "sff_id %u data %U", eeprom_info.sff_id, format_hexdump, + eeprom_info.buf, sizeof (eeprom_info.buf)); + } +done: + if (rv != VNET_DEV_OK) + oct_port_stop (vm, port); + return VNET_DEV_OK; +} + +void +oct_port_stop (vlib_main_t *vm, vnet_dev_port_t *port) +{ + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + int rrv; + + log_debug (port->dev, "port stop: port %u", port->port_id); + + vnet_dev_poll_port_remove (vm, port, oct_port_poll); + + rrv = roc_nix_npc_rx_ena_dis (nix, false); + if (rrv) + { + oct_roc_err (dev, rrv, "roc_nix_npc_rx_ena_dis() failed"); + return; + } + + foreach_vnet_dev_port_rx_queue (q, port) + oct_rxq_stop (vm, q); + + foreach_vnet_dev_port_tx_queue (q, port) + oct_txq_stop (vm, q); +} + +vnet_dev_rv_t +oct_port_cfg_change_precheck (vlib_main_t *vm, vnet_dev_port_t *port, + vnet_dev_port_cfg_change_req_t *req) +{ + vnet_dev_rv_t rv = VNET_DEV_OK; + + switch (req->type) + { + case VNET_DEV_PORT_CFG_MAX_RX_FRAME_SIZE: + if (port->started) + rv = VNET_DEV_ERR_PORT_STARTED; + break; + + case VNET_DEV_PORT_CFG_PROMISC_MODE: + case VNET_DEV_PORT_CFG_CHANGE_PRIMARY_HW_ADDR: + case VNET_DEV_PORT_CFG_ADD_SECONDARY_HW_ADDR: + case VNET_DEV_PORT_CFG_REMOVE_SECONDARY_HW_ADDR: + break; + + default: + rv = VNET_DEV_ERR_NOT_SUPPORTED; + }; + + return rv; +} + +vnet_dev_rv_t +oct_port_cfg_change (vlib_main_t *vm, vnet_dev_port_t *port, + vnet_dev_port_cfg_change_req_t *req) +{ + vnet_dev_rv_t rv = VNET_DEV_OK; + + switch (req->type) + { + case VNET_DEV_PORT_CFG_PROMISC_MODE: + { + } + break; + + case VNET_DEV_PORT_CFG_CHANGE_PRIMARY_HW_ADDR: + break; + + case VNET_DEV_PORT_CFG_ADD_SECONDARY_HW_ADDR: + break; + + case VNET_DEV_PORT_CFG_REMOVE_SECONDARY_HW_ADDR: + break; + + case VNET_DEV_PORT_CFG_MAX_RX_FRAME_SIZE: + break; + + default: + return VNET_DEV_ERR_NOT_SUPPORTED; + }; + + return rv; +} diff --git a/src/plugins/dev_octeon/queue.c b/src/plugins/dev_octeon/queue.c new file mode 100644 index 00000000000..9378fc3b7c7 --- /dev/null +++ b/src/plugins/dev_octeon/queue.c @@ -0,0 +1,303 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2023 Cisco Systems, Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +VLIB_REGISTER_LOG_CLASS (oct_log, static) = { + .class_name = "octeon", + .subclass_name = "queue", +}; + +static vnet_dev_rv_t +oct_roc_err (vnet_dev_t *dev, int rv, char *fmt, ...) +{ + u8 *s = 0; + va_list va; + + va_start (va, fmt); + s = va_format (s, fmt, &va); + va_end (va); + + log_err (dev, "%v - ROC error %s (%d)", s, roc_error_msg_get (rv), rv); + + vec_free (s); + return VNET_DEV_ERR_INTERNAL; +} + +vnet_dev_rv_t +oct_rx_queue_alloc (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq) +{ + vnet_dev_port_t *port = rxq->port; + vnet_dev_t *dev = port->dev; + + log_debug (dev, "rx_queue_alloc: queue %u alocated", rxq->queue_id); + return VNET_DEV_OK; +} + +void +oct_rx_queue_free (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq) +{ + vnet_dev_port_t *port = rxq->port; + vnet_dev_t *dev = port->dev; + + log_debug (dev, "rx_queue_free: queue %u", rxq->queue_id); +} + +vnet_dev_rv_t +oct_tx_queue_alloc (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) +{ + oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); + vnet_dev_port_t *port = txq->port; + vnet_dev_t *dev = port->dev; + + log_debug (dev, "tx_queue_alloc: queue %u alocated", txq->queue_id); + + return vnet_dev_dma_mem_alloc ( + vm, dev, sizeof (void *) * ROC_CN10K_NPA_BATCH_ALLOC_MAX_PTRS, 128, + (void **) &ctq->ba_buffer); +} + +void +oct_tx_queue_free (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) +{ + oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); + vnet_dev_port_t *port = txq->port; + vnet_dev_t *dev = port->dev; + + log_debug (dev, "tx_queue_free: queue %u", txq->queue_id); + + vnet_dev_dma_mem_free (vm, dev, ctq->ba_buffer); +} + +vnet_dev_rv_t +oct_rxq_init (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq) +{ + oct_rxq_t *crq = vnet_dev_get_rx_queue_data (rxq); + vnet_dev_t *dev = rxq->port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + vlib_buffer_pool_t *bp = + vlib_get_buffer_pool (vm, vnet_dev_get_rx_queue_buffer_pool_index (rxq)); + struct roc_nix *nix = cd->nix; + int rrv; + + struct npa_aura_s aura = {}; + struct npa_pool_s npapool = { .nat_align = 1 }; + + if ((rrv = roc_npa_pool_create (&crq->aura_handle, bp->alloc_size, rxq->size, + &aura, &npapool, 0))) + { + oct_rxq_deinit (vm, rxq); + return oct_roc_err (dev, rrv, "roc_npa_pool_create() failed"); + } + + crq->npa_pool_initialized = 1; + log_notice (dev, "NPA pool created, aura_handle = 0x%lx", crq->aura_handle); + + crq->cq = (struct roc_nix_cq){ + .nb_desc = rxq->size, + .qid = rxq->queue_id, + }; + + if ((rrv = roc_nix_cq_init (nix, &crq->cq))) + { + oct_rxq_deinit (vm, rxq); + return oct_roc_err (dev, rrv, + "roc_nix_cq_init(qid = %u, nb_desc = %u) failed", + crq->cq.nb_desc, crq->cq.nb_desc); + } + + crq->cq_initialized = 1; + log_debug (dev, "CQ %u initialised (qmask 0x%x wdata 0x%lx)", crq->cq.qid, + crq->cq.qmask, crq->cq.wdata); + + crq->hdr_off = vm->buffer_main->ext_hdr_size; + + crq->rq = (struct roc_nix_rq){ + .qid = rxq->queue_id, + .cqid = crq->cq.qid, + .aura_handle = crq->aura_handle, + .first_skip = crq->hdr_off + sizeof (vlib_buffer_t), + .later_skip = crq->hdr_off + sizeof (vlib_buffer_t), + .lpb_size = bp->data_size + crq->hdr_off + sizeof (vlib_buffer_t), + .flow_tag_width = 32, + }; + + if ((rrv = roc_nix_rq_init (nix, &crq->rq, 1 /* disable */))) + { + oct_rxq_deinit (vm, rxq); + return oct_roc_err (dev, rrv, "roc_nix_rq_init(qid = %u) failed", + crq->rq.qid); + } + + crq->rq_initialized = 1; + crq->lmt_base_addr = roc_idev_lmt_base_addr_get (); + crq->aura_batch_free_ioaddr = + (roc_npa_aura_handle_to_base (crq->aura_handle) + + NPA_LF_AURA_BATCH_FREE0) | + (0x7 << 4); + + log_debug (dev, "RQ %u initialised", crq->cq.qid); + + return VNET_DEV_OK; +} + +void +oct_rxq_deinit (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq) +{ + oct_rxq_t *crq = vnet_dev_get_rx_queue_data (rxq); + vnet_dev_t *dev = rxq->port->dev; + int rrv; + + if (crq->rq_initialized) + { + rrv = roc_nix_rq_fini (&crq->rq); + if (rrv) + oct_roc_err (dev, rrv, "roc_nix_rq_fini() failed"); + crq->rq_initialized = 0; + } + + if (crq->cq_initialized) + { + rrv = roc_nix_cq_fini (&crq->cq); + if (rrv) + oct_roc_err (dev, rrv, "roc_nix_cq_fini() failed"); + crq->cq_initialized = 0; + } + + if (crq->npa_pool_initialized) + { + rrv = roc_npa_pool_destroy (crq->aura_handle); + if (rrv) + oct_roc_err (dev, rrv, "roc_npa_pool_destroy() failed"); + crq->npa_pool_initialized = 0; + } +} + +vnet_dev_rv_t +oct_txq_init (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) +{ + oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); + vnet_dev_t *dev = txq->port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + struct npa_aura_s aura = {}; + struct npa_pool_s npapool = { .nat_align = 1 }; + int rrv; + vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, 0); + + if ((rrv = roc_npa_pool_create ( + &ctq->aura_handle, bp->alloc_size, + txq->size * 6 /* worst case - two SG with 3 segs each = 6 */, &aura, + &npapool, 0))) + { + oct_txq_deinit (vm, txq); + return oct_roc_err (dev, rrv, "roc_npa_pool_create() failed"); + } + + ctq->npa_pool_initialized = 1; + log_notice (dev, "NPA pool created, aura_handle = 0x%lx", ctq->aura_handle); + + ctq->sq = (struct roc_nix_sq){ + .nb_desc = txq->size, + .qid = txq->queue_id, + .max_sqe_sz = NIX_MAXSQESZ_W16, + }; + + if ((rrv = roc_nix_sq_init (nix, &ctq->sq))) + { + oct_txq_deinit (vm, txq); + return oct_roc_err ( + dev, rrv, + "roc_nix_sq_init(qid = %u, nb_desc = %u, max_sqe_sz = %u) failed", + ctq->sq.nb_desc, ctq->sq.max_sqe_sz); + } + + ctq->sq_initialized = 1; + log_debug (dev, "SQ initialised, qid %u, nb_desc %u, max_sqe_sz %u", + ctq->sq.qid, ctq->sq.nb_desc, ctq->sq.max_sqe_sz); + + ctq->hdr_off = vm->buffer_main->ext_hdr_size; + + if (ctq->sq.lmt_addr == 0) + ctq->sq.lmt_addr = (void *) nix->lmt_base; + ctq->io_addr = ctq->sq.io_addr & ~0x7fULL; + ctq->lmt_addr = ctq->sq.lmt_addr; + + return VNET_DEV_OK; +} + +void +oct_txq_deinit (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) +{ + oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); + vnet_dev_t *dev = txq->port->dev; + int rrv; + + if (ctq->sq_initialized) + { + rrv = roc_nix_sq_fini (&ctq->sq); + if (rrv) + oct_roc_err (dev, rrv, "roc_nix_sq_fini() failed"); + ctq->sq_initialized = 0; + } + + if (ctq->npa_pool_initialized) + { + rrv = roc_npa_pool_destroy (ctq->aura_handle); + if (rrv) + oct_roc_err (dev, rrv, "roc_npa_pool_destroy() failed"); + ctq->npa_pool_initialized = 0; + } +} + +u8 * +format_oct_rxq_info (u8 *s, va_list *args) +{ + vnet_dev_format_args_t *a = va_arg (*args, vnet_dev_format_args_t *); + vnet_dev_rx_queue_t *rxq = va_arg (*args, vnet_dev_rx_queue_t *); + oct_rxq_t *crq = vnet_dev_get_rx_queue_data (rxq); + u32 indent = format_get_indent (s); + + if (a->debug) + { + s = format (s, "n_enq %u cq_nb_desc %u", crq->n_enq, crq->cq.nb_desc); + s = format (s, "\n%Uaura: id 0x%x count %u limit %u avail %u", + format_white_space, indent, + roc_npa_aura_handle_to_aura (crq->aura_handle), + roc_npa_aura_op_cnt_get (crq->aura_handle), + roc_npa_aura_op_limit_get (crq->aura_handle), + roc_npa_aura_op_available (crq->aura_handle)); + } + return s; +} + +u8 * +format_oct_txq_info (u8 *s, va_list *args) +{ + vnet_dev_format_args_t *a = va_arg (*args, vnet_dev_format_args_t *); + vnet_dev_tx_queue_t *txq = va_arg (*args, vnet_dev_tx_queue_t *); + oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); + u32 indent = format_get_indent (s); + + if (a->debug) + { + s = format (s, "n_enq %u sq_nb_desc %u io_addr %p lmt_addr %p", + ctq->n_enq, ctq->sq.nb_desc, ctq->io_addr, ctq->lmt_addr); + s = format (s, "\n%Uaura: id 0x%x count %u limit %u avail %u", + format_white_space, indent, + roc_npa_aura_handle_to_aura (ctq->aura_handle), + roc_npa_aura_op_cnt_get (ctq->aura_handle), + roc_npa_aura_op_limit_get (ctq->aura_handle), + roc_npa_aura_op_available (ctq->aura_handle)); + } + + return s; +} diff --git a/src/plugins/dev_octeon/roc_helper.c b/src/plugins/dev_octeon/roc_helper.c new file mode 100644 index 00000000000..f10c2cb578b --- /dev/null +++ b/src/plugins/dev_octeon/roc_helper.c @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2023 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#include +#include +#include +#include +#include + +static oct_plt_memzone_list_t memzone_list; + +static inline void +oct_plt_log (oct_plt_log_level_t level, oct_plt_log_class_t cls, char *fmt, + ...) +{ + vlib_log ((vlib_log_level_t) level, cls, fmt); +} + +static inline void +oct_plt_spinlock_init (oct_plt_spinlock_t *p) +{ + clib_spinlock_init ((clib_spinlock_t *) p); +} + +static void +oct_plt_spinlock_lock (oct_plt_spinlock_t *p) +{ + clib_spinlock_lock ((clib_spinlock_t *) p); +} + +static void +oct_plt_spinlock_unlock (oct_plt_spinlock_t *p) +{ + clib_spinlock_unlock ((clib_spinlock_t *) p); +} + +static int +oct_plt_spinlock_trylock (oct_plt_spinlock_t *p) +{ + return clib_spinlock_trylock ((clib_spinlock_t *) p); +} + +static u64 +oct_plt_get_thread_index (void) +{ + return __os_thread_index; +} + +static void +oct_drv_physmem_free (vlib_main_t *vm, void *mem) +{ + if (!mem) + { + clib_warning ("Invalid address %p", mem); + return; + } + + vlib_physmem_free (vm, mem); +} + +static void * +oct_drv_physmem_alloc (vlib_main_t *vm, u32 size, u32 align) +{ + clib_error_t *error = NULL; + uword *mem = NULL; + + if (align) + { + /* Force cache line alloc in case alignment is less than cache line */ + align = align < CLIB_CACHE_LINE_BYTES ? CLIB_CACHE_LINE_BYTES : align; + mem = vlib_physmem_alloc_aligned_on_numa (vm, size, align, 0); + } + else + mem = + vlib_physmem_alloc_aligned_on_numa (vm, size, CLIB_CACHE_LINE_BYTES, 0); + if (!mem) + return NULL; + + error = vfio_map_physmem_page (vm, mem); + if (error) + goto report_error; + + clib_memset (mem, 0, size); + return mem; + +report_error: + clib_error_report (error); + oct_drv_physmem_free (vm, mem); + + return NULL; +} + +static void +oct_plt_free (void *addr) +{ + vlib_main_t *vm = vlib_get_main (); + + oct_drv_physmem_free ((void *) vm, addr); +} + +static void * +oct_plt_zmalloc (u32 size, u32 align) +{ + vlib_main_t *vm = vlib_get_main (); + + return oct_drv_physmem_alloc (vm, size, align); +} + +static oct_plt_memzone_t * +memzone_get (u32 index) +{ + if (index == ((u32) ~0)) + return 0; + + return pool_elt_at_index (memzone_list.mem_pool, index); +} + +static int +oct_plt_memzone_free (const oct_plt_memzone_t *name) +{ + uword *p; + p = hash_get_mem (memzone_list.memzone_by_name, name); + + if (p[0] == ((u32) ~0)) + return -EINVAL; + + hash_unset_mem (memzone_list.memzone_by_name, name); + + pool_put_index (memzone_list.mem_pool, p[0]); + + return 0; +} + +static oct_plt_memzone_t * +oct_plt_memzone_lookup (const char *name) +{ + uword *p; + p = hash_get_mem (memzone_list.memzone_by_name, name); + if (p) + return memzone_get (p[0]); + + return 0; +} + +static oct_plt_memzone_t * +oct_plt_memzone_reserve_aligned (const char *name, u64 len, u8 socket, + u32 flags, u32 align) +{ + oct_plt_memzone_t *mem_pool; + void *p = NULL; + + pool_get_zero (memzone_list.mem_pool, mem_pool); + + p = oct_plt_zmalloc (len, align); + if (!p) + return NULL; + + mem_pool->addr = p; + mem_pool->index = mem_pool - memzone_list.mem_pool; + hash_set_mem (memzone_list.memzone_by_name, name, mem_pool->index); + + return mem_pool; +} + +oct_plt_init_param_t oct_plt_init_param = { + .oct_plt_log_reg_class = vlib_log_register_class, + .oct_plt_log = oct_plt_log, + .oct_plt_free = oct_plt_free, + .oct_plt_zmalloc = oct_plt_zmalloc, + .oct_plt_memzone_free = oct_plt_memzone_free, + .oct_plt_memzone_lookup = oct_plt_memzone_lookup, + .oct_plt_memzone_reserve_aligned = oct_plt_memzone_reserve_aligned, + .oct_plt_spinlock_init = oct_plt_spinlock_init, + .oct_plt_spinlock_lock = oct_plt_spinlock_lock, + .oct_plt_spinlock_unlock = oct_plt_spinlock_unlock, + .oct_plt_spinlock_trylock = oct_plt_spinlock_trylock, + .oct_plt_get_thread_index = oct_plt_get_thread_index, +}; diff --git a/src/plugins/dev_octeon/rx_node.c b/src/plugins/dev_octeon/rx_node.c new file mode 100644 index 00000000000..c1c4771795e --- /dev/null +++ b/src/plugins/dev_octeon/rx_node.c @@ -0,0 +1,385 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2023 Cisco Systems, Inc. + */ + +#include +#include +#include +#include +#include + +typedef struct +{ + u32 next_index; + u32 sw_if_index; + u32 hw_if_index; + u32 trace_count; + u32 n_traced; + oct_nix_rx_cqe_desc_t *next_desc; + u64 parse_w0_or; + u32 n_left_to_next; + u32 *to_next; + u32 n_rx_pkts; + u32 n_rx_bytes; + u32 n_segs; +} oct_rx_node_ctx_t; + +static_always_inline vlib_buffer_t * +oct_seg_to_bp (void *p) +{ + return (vlib_buffer_t *) p - 1; +} + +static_always_inline void +oct_rx_attach_tail (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, vlib_buffer_t *h, + oct_nix_rx_cqe_desc_t *d) +{ + u32 tail_sz = 0, n_tail_segs = 0; + vlib_buffer_t *p, *b; + u8 segs0 = d->sg0.segs, segs1 = 0; + + if (segs0 < 2) + return; + + b = oct_seg_to_bp (d->segs0[1]); + h->next_buffer = vlib_get_buffer_index (vm, b); + tail_sz += b->current_length = d->sg0.seg2_size; + n_tail_segs++; + + if (segs0 == 2) + goto done; + + p = b; + p->flags = VLIB_BUFFER_NEXT_PRESENT; + b = oct_seg_to_bp (d->segs0[2]); + p->next_buffer = vlib_get_buffer_index (vm, b); + tail_sz += b->current_length = d->sg0.seg3_size; + n_tail_segs++; + + if (d->sg1.subdc != NIX_SUBDC_SG) + goto done; + + segs1 = d->sg1.segs; + if (segs1 == 0) + goto done; + + p = b; + p->flags = VLIB_BUFFER_NEXT_PRESENT; + b = oct_seg_to_bp (d->segs1[0]); + p->next_buffer = vlib_get_buffer_index (vm, b); + tail_sz += b->current_length = d->sg1.seg1_size; + n_tail_segs++; + + if (segs1 == 1) + goto done; + + p = b; + p->flags = VLIB_BUFFER_NEXT_PRESENT; + b = oct_seg_to_bp (d->segs1[1]); + p->next_buffer = vlib_get_buffer_index (vm, b); + tail_sz += b->current_length = d->sg1.seg2_size; + n_tail_segs++; + + if (segs1 == 2) + goto done; + + p = b; + p->flags = VLIB_BUFFER_NEXT_PRESENT; + b = oct_seg_to_bp (d->segs1[2]); + p->next_buffer = vlib_get_buffer_index (vm, b); + tail_sz += b->current_length = d->sg1.seg3_size; + n_tail_segs++; + +done: + b->flags = 0; + h->total_length_not_including_first_buffer = tail_sz; + h->flags |= VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID; + ctx->n_rx_bytes += tail_sz; + ctx->n_segs += n_tail_segs; +} + +static_always_inline u32 +oct_rx_batch (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, + vnet_dev_rx_queue_t *rxq, u32 n) +{ + oct_rxq_t *crq = vnet_dev_get_rx_queue_data (rxq); + vlib_buffer_template_t bt = rxq->buffer_template; + u32 n_left; + oct_nix_rx_cqe_desc_t *d = ctx->next_desc; + vlib_buffer_t *b[4]; + + for (n_left = n; n_left >= 8; d += 4, n_left -= 4, ctx->to_next += 4) + { + u32 segs = 0; + clib_prefetch_store (oct_seg_to_bp (d[4].segs0[0])); + clib_prefetch_store (oct_seg_to_bp (d[5].segs0[0])); + b[0] = oct_seg_to_bp (d[0].segs0[0]); + clib_prefetch_store (oct_seg_to_bp (d[6].segs0[0])); + b[1] = oct_seg_to_bp (d[1].segs0[0]); + clib_prefetch_store (oct_seg_to_bp (d[7].segs0[0])); + b[2] = oct_seg_to_bp (d[2].segs0[0]); + b[3] = oct_seg_to_bp (d[3].segs0[0]); + ctx->to_next[0] = vlib_get_buffer_index (vm, b[0]); + ctx->to_next[1] = vlib_get_buffer_index (vm, b[1]); + ctx->to_next[2] = vlib_get_buffer_index (vm, b[2]); + ctx->to_next[3] = vlib_get_buffer_index (vm, b[3]); + b[0]->template = bt; + b[1]->template = bt; + b[2]->template = bt; + b[3]->template = bt; + ctx->n_rx_bytes += b[0]->current_length = d[0].sg0.seg1_size; + ctx->n_rx_bytes += b[1]->current_length = d[1].sg0.seg1_size; + ctx->n_rx_bytes += b[2]->current_length = d[2].sg0.seg1_size; + ctx->n_rx_bytes += b[3]->current_length = d[3].sg0.seg1_size; + ctx->n_segs += 4; + segs = d[0].sg0.segs + d[1].sg0.segs + d[2].sg0.segs + d[3].sg0.segs; + + if (PREDICT_FALSE (segs > 4)) + { + oct_rx_attach_tail (vm, ctx, b[0], d + 0); + oct_rx_attach_tail (vm, ctx, b[1], d + 1); + oct_rx_attach_tail (vm, ctx, b[2], d + 2); + oct_rx_attach_tail (vm, ctx, b[3], d + 3); + } + } + + for (; n_left; d += 1, n_left -= 1, ctx->to_next += 1) + { + b[0] = (vlib_buffer_t *) d->segs0[0] - 1; + ctx->to_next[0] = vlib_get_buffer_index (vm, b[0]); + b[0]->template = bt; + ctx->n_rx_bytes += b[0]->current_length = d[0].sg0.seg1_size; + ctx->n_segs += 1; + if (d[0].sg0.segs > 1) + oct_rx_attach_tail (vm, ctx, b[0], d + 0); + } + + plt_write64 ((crq->cq.wdata | n), crq->cq.door); + ctx->n_rx_pkts += n; + ctx->n_left_to_next -= n; + return n; +} + +static_always_inline void +oct_rxq_refill_batch (vlib_main_t *vm, u64 lmt_id, u64 addr, + oct_npa_lf_aura_batch_free_line_t *lines, u32 *bi, + oct_npa_lf_aura_batch_free0_t w0, u64 n_lines) +{ + u64 data; + + for (u32 i = 0; i < n_lines; i++, bi += 15) + { + lines[i].w0 = w0; + vlib_get_buffers (vm, bi, (vlib_buffer_t **) lines[i].data, 15); + } + + data = lmt_id | ((n_lines - 1) << 12) | ((1ULL << (n_lines * 3)) - 1) << 19; + roc_lmt_submit_steorl (data, addr); + + /* Data Store Memory Barrier - outer shareable domain */ + asm volatile("dmb oshst" ::: "memory"); +} + +static_always_inline u32 +oct_rxq_refill (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u16 n_refill) +{ + const u32 batch_max_lines = 16; + const u32 bufs_per_line = 15; + const u32 batch_max_bufs = 15 * 16; + + u32 batch_bufs, n_lines, n_alloc; + u32 buffer_indices[batch_max_bufs]; + u64 lmt_addr, lmt_id, addr, n_enq = 0; + u8 bpi = vnet_dev_get_rx_queue_buffer_pool_index (rxq); + oct_rxq_t *crq = vnet_dev_get_rx_queue_data (rxq); + oct_npa_lf_aura_batch_free_line_t *lines; + + if (n_refill < bufs_per_line) + return 0; + + n_lines = n_refill / bufs_per_line; + + addr = crq->aura_batch_free_ioaddr; + lmt_addr = crq->lmt_base_addr; + lmt_id = vm->thread_index << ROC_LMT_LINES_PER_CORE_LOG2; + lmt_addr += lmt_id << ROC_LMT_LINE_SIZE_LOG2; + lines = (oct_npa_lf_aura_batch_free_line_t *) lmt_addr; + + oct_npa_lf_aura_batch_free0_t w0 = { + .aura = roc_npa_aura_handle_to_aura (crq->aura_handle), + .count_eot = 1, + }; + + while (n_lines >= batch_max_lines) + { + n_alloc = vlib_buffer_alloc (vm, buffer_indices, batch_max_bufs); + if (PREDICT_FALSE (n_alloc < batch_max_bufs)) + goto alloc_fail; + oct_rxq_refill_batch (vm, lmt_id, addr, lines, buffer_indices, w0, + batch_max_lines); + n_lines -= batch_max_lines; + n_enq += batch_max_bufs; + } + + if (n_lines == 0) + return n_enq; + + batch_bufs = n_lines * bufs_per_line; + n_alloc = vlib_buffer_alloc_from_pool (vm, buffer_indices, batch_bufs, bpi); + + if (PREDICT_FALSE (n_alloc < batch_bufs)) + { + alloc_fail: + if (n_alloc >= bufs_per_line) + { + u32 n_unalloc; + n_lines = n_alloc / bufs_per_line; + batch_bufs = n_lines * bufs_per_line; + n_unalloc = n_alloc - batch_bufs; + + if (n_unalloc) + vlib_buffer_unalloc_to_pool (vm, buffer_indices + batch_bufs, + n_unalloc, bpi); + } + else + { + if (n_alloc) + vlib_buffer_unalloc_to_pool (vm, buffer_indices, n_alloc, bpi); + return n_enq; + } + } + + oct_rxq_refill_batch (vm, lmt_id, addr, lines, buffer_indices, w0, n_lines); + n_enq += batch_bufs; + + return n_enq; +} + +static_always_inline void +oct_rx_trace (vlib_main_t *vm, vlib_node_runtime_t *node, + oct_rx_node_ctx_t *ctx, oct_nix_rx_cqe_desc_t *d, u32 n_desc) +{ + u32 i = 0; + if (PREDICT_TRUE (ctx->trace_count == 0)) + return; + + while (ctx->n_traced < ctx->trace_count && i < n_desc) + { + vlib_buffer_t *b = (vlib_buffer_t *) d[i].segs0[0] - 1; + + if (PREDICT_TRUE (vlib_trace_buffer (vm, node, ctx->next_index, b, + /* follow_chain */ 0))) + { + oct_rx_trace_t *tr = vlib_add_trace (vm, node, b, sizeof (*tr)); + tr->next_index = ctx->next_index; + tr->sw_if_index = ctx->sw_if_index; + tr->desc = d[i]; + ctx->n_traced++; + } + i++; + } +} + +static_always_inline uword +oct_rx_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_frame_t *frame, vnet_dev_port_t *port, + vnet_dev_rx_queue_t *rxq, int with_flows) +{ + vnet_main_t *vnm = vnet_get_main (); + u32 thr_idx = vlib_get_thread_index (); + oct_rxq_t *crq = vnet_dev_get_rx_queue_data (rxq); + u32 n_desc, head, n, n_enq; + u32 cq_size = crq->cq.nb_desc; + u32 cq_mask = crq->cq.qmask; + oct_nix_rx_cqe_desc_t *descs = crq->cq.desc_base; + oct_nix_lf_cq_op_status_t status; + oct_rx_node_ctx_t _ctx = { + .next_index = rxq->next_index, + .sw_if_index = port->intf.sw_if_index, + .hw_if_index = port->intf.hw_if_index, + }, *ctx = &_ctx; + + /* get head and tail from NIX_LF_CQ_OP_STATUS */ + status.as_u64 = roc_atomic64_add_sync (crq->cq.wdata, crq->cq.status); + if (status.cq_err || status.op_err) + return 0; + + head = status.head; + n_desc = (status.tail - head) & cq_mask; + + if (n_desc == 0) + return 0; + + vlib_get_new_next_frame (vm, node, ctx->next_index, ctx->to_next, + ctx->n_left_to_next); + + ctx->trace_count = vlib_get_trace_count (vm, node); + + while (1) + { + ctx->next_desc = descs + head; + n = clib_min (cq_size - head, clib_min (n_desc, ctx->n_left_to_next)); + n = oct_rx_batch (vm, ctx, rxq, n); + oct_rx_trace (vm, node, ctx, descs + head, n); + + if (ctx->n_left_to_next == 0) + break; + + status.as_u64 = roc_atomic64_add_sync (crq->cq.wdata, crq->cq.status); + if (status.cq_err || status.op_err) + break; + + head = status.head; + n_desc = (status.tail - head) & cq_mask; + if (n_desc == 0) + break; + } + + if (ctx->n_traced) + vlib_set_trace_count (vm, node, ctx->trace_count - ctx->n_traced); + + if (PREDICT_TRUE (ctx->next_index == VNET_DEV_ETH_RX_PORT_NEXT_ETH_INPUT)) + { + vlib_next_frame_t *nf; + vlib_frame_t *f; + ethernet_input_frame_t *ef; + oct_nix_rx_parse_t p = { .w[0] = ctx->parse_w0_or }; + nf = vlib_node_runtime_get_next_frame (vm, node, ctx->next_index); + f = vlib_get_frame (vm, nf->frame); + f->flags = ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX; + + ef = vlib_frame_scalar_args (f); + ef->sw_if_index = ctx->sw_if_index; + ef->hw_if_index = ctx->hw_if_index; + + if (p.f.errcode == 0 && p.f.errlev == 0) + f->flags |= ETH_INPUT_FRAME_F_IP4_CKSUM_OK; + + vlib_frame_no_append (f); + } + + vlib_put_next_frame (vm, node, ctx->next_index, ctx->n_left_to_next); + + n_enq = crq->n_enq - ctx->n_segs; + n_enq += oct_rxq_refill (vm, rxq, rxq->size - n_enq); + crq->n_enq = n_enq; + + vlib_increment_combined_counter ( + vnm->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, + thr_idx, ctx->hw_if_index, ctx->n_rx_pkts, ctx->n_rx_bytes); + + return ctx->n_rx_pkts; +} + +VNET_DEV_NODE_FN (oct_rx_node) +(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame) +{ + u32 n_rx = 0; + foreach_vnet_dev_rx_queue_runtime (rxq, node) + { + vnet_dev_port_t *port = rxq->port; + n_rx += oct_rx_node_inline (vm, node, frame, port, rxq, 0); + } + + return n_rx; +} diff --git a/src/plugins/dev_octeon/tx_node.c b/src/plugins/dev_octeon/tx_node.c new file mode 100644 index 00000000000..5deaa82a0c1 --- /dev/null +++ b/src/plugins/dev_octeon/tx_node.c @@ -0,0 +1,427 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2023 Cisco Systems, Inc. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +typedef struct +{ + union nix_send_hdr_w0_u hdr_w0_teplate; + vlib_node_runtime_t *node; + u32 n_tx_bytes; + u32 n_drop; + vlib_buffer_t *drop[VLIB_FRAME_SIZE]; + u32 batch_alloc_not_ready; + u32 batch_alloc_issue_fail; + u16 lmt_id; + u64 lmt_ioaddr; + lmt_line_t *lmt_lines; +} oct_tx_ctx_t; + +static_always_inline u32 +oct_batch_free (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq) +{ + oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); + u8 num_cl; + u64 ah; + u32 n_freed = 0, n; + oct_npa_batch_alloc_cl128_t *cl; + + num_cl = ctq->ba_num_cl; + if (num_cl) + { + u16 off = ctq->hdr_off; + u32 *bi = (u32 *) ctq->ba_buffer; + + for (cl = ctq->ba_buffer + ctq->ba_first_cl; num_cl > 0; num_cl--, cl++) + { + u8 count; + if (cl->status.ccode == ALLOC_CCODE_INVAL) + { + ctx->batch_alloc_not_ready++; + n_freed = bi - (u32 *) ctq->ba_buffer; + if (n_freed > 0) + { + vlib_buffer_free_no_next (vm, (u32 *) ctq->ba_buffer, + n_freed); + ctq->ba_num_cl = num_cl; + ctq->ba_first_cl = cl - ctq->ba_buffer; + return n_freed; + } + + return 0; + } + + count = cl->status.count; +#if (CLIB_DEBUG > 0) + cl->status.count = cl->status.ccode = 0; +#endif + if (PREDICT_TRUE (count == 16)) + { + /* optimize for likely case where cacheline is full */ + vlib_get_buffer_indices_with_offset (vm, (void **) cl, bi, 16, + off); + bi += 16; + } + else + { + vlib_get_buffer_indices_with_offset (vm, (void **) cl, bi, count, + off); + bi += count; + } + } + + n_freed = bi - (u32 *) ctq->ba_buffer; + if (n_freed > 0) + vlib_buffer_free_no_next (vm, (u32 *) ctq->ba_buffer, n_freed); + + /* clear status bits in each cacheline */ + n = cl - ctq->ba_buffer; + for (u32 i = 0; i < n; i++) + ctq->ba_buffer[i].iova[0] = 0; + + ctq->ba_num_cl = ctq->ba_first_cl = 0; + } + + ah = ctq->aura_handle; + + if ((n = roc_npa_aura_op_available (ah)) >= 32) + { + u64 addr, res; + + n = clib_min (n, ROC_CN10K_NPA_BATCH_ALLOC_MAX_PTRS); + + oct_npa_batch_alloc_compare_t cmp = { + .compare_s = { .aura = roc_npa_aura_handle_to_aura (ah), + .stype = ALLOC_STYPE_STF, + .count = n } + }; + + addr = roc_npa_aura_handle_to_base (ah) + NPA_LF_AURA_BATCH_ALLOC; + res = roc_atomic64_casl (cmp.as_u64, (uint64_t) ctq->ba_buffer, + (i64 *) addr); + if (res == ALLOC_RESULT_ACCEPTED || res == ALLOC_RESULT_NOCORE) + { + ctq->ba_num_cl = (n + 15) / 16; + ctq->ba_first_cl = 0; + } + else + ctx->batch_alloc_issue_fail++; + } + + return n_freed; +} + +static_always_inline u8 +oct_tx_enq1 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vlib_buffer_t *b, + lmt_line_t *line, u32 flags, int simple, int trace) +{ + u8 n_dwords = 2; + u32 total_len = 0; + oct_tx_desc_t d = { + .hdr_w0 = ctx->hdr_w0_teplate, + .sg[0] = { + .segs = 1, + .subdc = NIX_SUBDC_SG, + }, + .sg[4] = { + .subdc = NIX_SUBDC_SG, + }, + }; + + if (!simple && flags & VLIB_BUFFER_NEXT_PRESENT) + { + u8 n_tail_segs = 0; + vlib_buffer_t *tail_segs[5], *t = b; + + while (t->flags & VLIB_BUFFER_NEXT_PRESENT) + { + t = vlib_get_buffer (vm, t->next_buffer); + tail_segs[n_tail_segs++] = t; + if (n_tail_segs > 5) + { + ctx->drop[ctx->n_drop++] = t; + return 0; + } + } + + switch (n_tail_segs) + { + case 5: + d.sg[7].u = (u64) vlib_buffer_get_current (tail_segs[4]); + total_len += d.sg[4].seg3_size = tail_segs[4]->current_length; + d.sg[4].segs++; + case 4: + d.sg[6].u = (u64) vlib_buffer_get_current (tail_segs[3]); + total_len += d.sg[4].seg2_size = tail_segs[3]->current_length; + d.sg[4].segs++; + n_dwords++; + case 3: + d.sg[5].u = (u64) vlib_buffer_get_current (tail_segs[2]); + total_len += d.sg[4].seg1_size = tail_segs[2]->current_length; + d.sg[4].segs++; + n_dwords++; + case 2: + d.sg[3].u = (u64) vlib_buffer_get_current (tail_segs[1]); + total_len += d.sg[0].seg3_size = tail_segs[1]->current_length; + d.sg[0].segs++; + case 1: + d.sg[2].u = (u64) vlib_buffer_get_current (tail_segs[0]); + total_len += d.sg[0].seg2_size = tail_segs[0]->current_length; + d.sg[0].segs++; + n_dwords++; + default: + break; + }; + d.hdr_w0.sizem1 = n_dwords - 1; + } + + if (!simple && flags & VNET_BUFFER_F_OFFLOAD) + { + vnet_buffer_oflags_t oflags = vnet_buffer (b)->oflags; + if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM) + { + d.hdr_w1.ol3type = NIX_SENDL3TYPE_IP4_CKSUM; + d.hdr_w1.ol3ptr = vnet_buffer (b)->l3_hdr_offset; + d.hdr_w1.ol4ptr = + vnet_buffer (b)->l3_hdr_offset + sizeof (ip4_header_t); + } + if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM) + { + d.hdr_w1.ol4type = NIX_SENDL4TYPE_UDP_CKSUM; + d.hdr_w1.ol4ptr = vnet_buffer (b)->l4_hdr_offset; + } + else if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM) + { + d.hdr_w1.ol4type = NIX_SENDL4TYPE_TCP_CKSUM; + d.hdr_w1.ol4ptr = vnet_buffer (b)->l4_hdr_offset; + } + } + + total_len += d.sg[0].seg1_size = b->current_length; + d.hdr_w0.total = total_len; + d.sg[1].u = (u64) vlib_buffer_get_current (b); + + if (trace && flags & VLIB_BUFFER_IS_TRACED) + { + oct_tx_trace_t *t = vlib_add_trace (vm, ctx->node, b, sizeof (*t)); + t->desc = d; + t->sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_TX]; + } + + for (u32 i = 0; i < n_dwords; i++) + line->dwords[i] = d.as_u128[i]; + + return n_dwords; +} + +static_always_inline u32 +oct_tx_enq16 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq, + vlib_buffer_t **b, u32 n_pkts, int trace) +{ + u8 dwords_per_line[16], *dpl = dwords_per_line; + u64 lmt_arg, ioaddr, n_lines; + u32 n_left, or_flags_16 = 0; + const u32 not_simple_flags = + VLIB_BUFFER_NEXT_PRESENT | VNET_BUFFER_F_OFFLOAD; + lmt_line_t *l = ctx->lmt_lines; + + /* Data Store Memory Barrier - outer shareable domain */ + asm volatile("dmb oshst" ::: "memory"); + + for (n_left = n_pkts; n_left >= 8; n_left -= 8, b += 8, l += 8) + { + u32 f0, f1, f2, f3, f4, f5, f6, f7, or_f = 0; + vlib_prefetch_buffer_header (b[8], LOAD); + or_f |= f0 = b[0]->flags; + or_f |= f1 = b[1]->flags; + vlib_prefetch_buffer_header (b[9], LOAD); + or_f |= f2 = b[2]->flags; + or_f |= f3 = b[3]->flags; + vlib_prefetch_buffer_header (b[10], LOAD); + or_f |= f4 = b[4]->flags; + or_f |= f5 = b[5]->flags; + vlib_prefetch_buffer_header (b[11], LOAD); + or_f |= f6 = b[6]->flags; + or_f |= f7 = b[7]->flags; + vlib_prefetch_buffer_header (b[12], LOAD); + or_flags_16 |= or_f; + + if ((or_f & not_simple_flags) == 0) + { + int simple = 1; + oct_tx_enq1 (vm, ctx, b[0], l, f0, simple, trace); + oct_tx_enq1 (vm, ctx, b[1], l + 1, f1, simple, trace); + vlib_prefetch_buffer_header (b[13], LOAD); + oct_tx_enq1 (vm, ctx, b[2], l + 2, f2, simple, trace); + oct_tx_enq1 (vm, ctx, b[3], l + 3, f3, simple, trace); + vlib_prefetch_buffer_header (b[14], LOAD); + oct_tx_enq1 (vm, ctx, b[4], l + 4, f4, simple, trace); + oct_tx_enq1 (vm, ctx, b[5], l + 5, f5, simple, trace); + vlib_prefetch_buffer_header (b[15], LOAD); + oct_tx_enq1 (vm, ctx, b[6], l + 6, f6, simple, trace); + oct_tx_enq1 (vm, ctx, b[7], l + 7, f7, simple, trace); + dpl[0] = dpl[1] = dpl[2] = dpl[3] = 2; + dpl[4] = dpl[5] = dpl[6] = dpl[7] = 2; + } + else + { + int simple = 0; + dpl[0] = oct_tx_enq1 (vm, ctx, b[0], l, f0, simple, trace); + dpl[1] = oct_tx_enq1 (vm, ctx, b[1], l + 1, f1, simple, trace); + vlib_prefetch_buffer_header (b[13], LOAD); + dpl[2] = oct_tx_enq1 (vm, ctx, b[2], l + 2, f2, simple, trace); + dpl[3] = oct_tx_enq1 (vm, ctx, b[3], l + 3, f3, simple, trace); + vlib_prefetch_buffer_header (b[14], LOAD); + dpl[4] = oct_tx_enq1 (vm, ctx, b[4], l + 4, f4, simple, trace); + dpl[5] = oct_tx_enq1 (vm, ctx, b[5], l + 5, f5, simple, trace); + vlib_prefetch_buffer_header (b[15], LOAD); + dpl[6] = oct_tx_enq1 (vm, ctx, b[6], l + 6, f6, simple, trace); + dpl[7] = oct_tx_enq1 (vm, ctx, b[7], l + 7, f7, simple, trace); + } + dpl += 8; + } + + for (; n_left > 0; n_left -= 1, b += 1, l += 1) + { + u32 f0 = b[0]->flags; + dpl++[0] = oct_tx_enq1 (vm, ctx, b[0], l, f0, 0, trace); + or_flags_16 |= f0; + } + + lmt_arg = ctx->lmt_id; + ioaddr = ctx->lmt_ioaddr; + n_lines = n_pkts; + + if (PREDICT_FALSE (or_flags_16 & VLIB_BUFFER_NEXT_PRESENT)) + { + dpl = dwords_per_line; + ioaddr |= (dpl[0] - 1) << 4; + + if (n_lines > 1) + { + lmt_arg |= (--n_lines) << 12; + + for (u8 bit_off = 19; n_lines; n_lines--, bit_off += 3, dpl++) + lmt_arg |= ((u64) dpl[1] - 1) << bit_off; + } + } + else + { + const u64 n_dwords = 2; + ioaddr |= (n_dwords - 1) << 4; + + if (n_lines > 1) + { + lmt_arg |= (--n_lines) << 12; + + for (u8 bit_off = 19; n_lines; n_lines--, bit_off += 3) + lmt_arg |= (n_dwords - 1) << bit_off; + } + } + + roc_lmt_submit_steorl (lmt_arg, ioaddr); + + return n_pkts; +} + +VNET_DEV_NODE_FN (oct_tx_node) +(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame) +{ + vnet_dev_tx_node_runtime_t *rt = vnet_dev_get_tx_node_runtime (node); + vnet_dev_tx_queue_t *txq = rt->tx_queue; + oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); + u32 node_index = node->node_index; + u32 *from = vlib_frame_vector_args (frame); + u32 n, n_enq, n_left, n_pkts = frame->n_vectors; + vlib_buffer_t *buffers[VLIB_FRAME_SIZE + 8], **b = buffers; + u64 lmt_id = vm->thread_index << ROC_LMT_LINES_PER_CORE_LOG2; + + oct_tx_ctx_t ctx = { + .node = node, + .hdr_w0_teplate = { + .aura = roc_npa_aura_handle_to_aura (ctq->aura_handle), + .sq = ctq->sq.qid, + .sizem1 = 1, + }, + .lmt_id = lmt_id, + .lmt_ioaddr = ctq->io_addr, + .lmt_lines = ctq->lmt_addr + (lmt_id << ROC_LMT_LINE_SIZE_LOG2), + }; + + vlib_get_buffers (vm, vlib_frame_vector_args (frame), b, n_pkts); + for (int i = 0; i < 8; i++) + b[n_pkts + i] = b[n_pkts - 1]; + + vnet_dev_tx_queue_lock_if_needed (txq); + + n_enq = ctq->n_enq; + n_enq -= oct_batch_free (vm, &ctx, txq); + + if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE)) + { + for (n_left = clib_min (n_pkts, txq->size - n_enq), n = 0; n_left >= 16; + n_left -= 16, b += 16) + n += oct_tx_enq16 (vm, &ctx, txq, b, 16, /* trace */ 1); + + if (n_left) + n += oct_tx_enq16 (vm, &ctx, txq, b, n_left, /* trace */ 1); + } + else + { + for (n_left = clib_min (n_pkts, txq->size - n_enq), n = 0; n_left >= 16; + n_left -= 16, b += 16) + n += oct_tx_enq16 (vm, &ctx, txq, b, 16, /* trace */ 0); + + if (n_left) + n += oct_tx_enq16 (vm, &ctx, txq, b, n_left, /* trace */ 0); + } + + ctq->n_enq = n_enq + n; + + if (n < n_pkts) + { + n = n_pkts - n; + vlib_buffer_free (vm, from + n, n); + vlib_error_count (vm, node->node_index, OCT_TX_NODE_CTR_NO_FREE_SLOTS, + n); + n_pkts -= ctx.n_drop; + } + + if (ctx.n_drop) + vlib_error_count (vm, node->node_index, OCT_TX_NODE_CTR_CHAIN_TOO_LONG, + ctx.n_drop); + + if (ctx.batch_alloc_not_ready) + vlib_error_count (vm, node_index, + OCT_TX_NODE_CTR_AURA_BATCH_ALLOC_NOT_READY, + ctx.batch_alloc_not_ready); + + if (ctx.batch_alloc_issue_fail) + vlib_error_count (vm, node_index, + OCT_TX_NODE_CTR_AURA_BATCH_ALLOC_ISSUE_FAIL, + ctx.batch_alloc_issue_fail); + + vnet_dev_tx_queue_unlock_if_needed (txq); + + if (ctx.n_drop) + { + u32 bi[VLIB_FRAME_SIZE]; + vlib_get_buffer_indices (vm, ctx.drop, bi, ctx.n_drop); + vlib_buffer_free (vm, bi, ctx.n_drop); + n_pkts -= ctx.n_drop; + } + + return n_pkts; +}