From e3d75c30a18637ce8ce118e93fb3e7e90422c9be Mon Sep 17 00:00:00 2001 From: Asaf Sonis Date: Tue, 5 Sep 2023 12:04:51 +0300 Subject: [PATCH] issue: 3593939 Remove DPCP dependency from VMA Signed-off-by: Asaf Sonis --- .ci/matrix_job.yaml | 1 - config/m4/dpcp.m4 | 80 -------------------- configure.ac | 2 - contrib/jenkins_tests/globals.sh | 49 ------------ contrib/test_jenkins.sh | 9 --- src/vma/Makefile.am | 2 +- src/vma/dev/ib_ctx_handler.cpp | 123 ------------------------------- src/vma/dev/ib_ctx_handler.h | 11 --- src/vma/sock/sock-redirect.cpp | 42 ----------- src/vma/vma_extra.h | 18 ----- 10 files changed, 1 insertion(+), 336 deletions(-) delete mode 100644 config/m4/dpcp.m4 diff --git a/.ci/matrix_job.yaml b/.ci/matrix_job.yaml index 13951435c..16db4812f 100644 --- a/.ci/matrix_job.yaml +++ b/.ci/matrix_job.yaml @@ -50,7 +50,6 @@ matrix: flags: - default - extra - - dpcp arch: - x86_64 - aarch64 diff --git a/config/m4/dpcp.m4 b/config/m4/dpcp.m4 deleted file mode 100644 index ed9b46575..000000000 --- a/config/m4/dpcp.m4 +++ /dev/null @@ -1,80 +0,0 @@ -# dpcp.m4 - Library to operate with DevX -# -# Copyright (c) 2001-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# See file LICENSE for terms. -# - -########################## -# libdpcp usage support -# -AC_DEFUN([DPCP_CAPABILITY_SETUP], -[ -AC_ARG_WITH([dpcp], - AS_HELP_STRING([--with-dpcp(=DIR)], - [Search for dpcp headers and libraries in DIR (default NO)]), - [], - [with_dpcp=no] -) - -if test "x$vma_cv_directverbs" != x3 && test "x$with_dpcp" != xno; then - AC_MSG_ERROR([dpcp can be used under RDMA-core subsystem only]) -fi - -vma_cv_dpcp=0 -AS_IF([test "x$with_dpcp" == xno], - [], - [ - if test -z "$with_dpcp" || test "$with_dpcp" = "yes"; then - with_dpcp=/usr - fi - - FUNC_CHECK_WITHDIR([dpcp], [$with_dpcp], [include/mellanox/dpcp.h]) - - vma_cv_dpcp_save_CPPFLAGS="$CPPFLAGS" - vma_cv_dpcp_save_CXXFLAGS="$CXXFLAGS" - vma_cv_dpcp_save_CFLAGS="$CFLAGS" - vma_cv_dpcp_save_LDFLAGS="$LDFLAGS" - vma_cv_dpcp_save_LIBS="$LIBS" - - vma_cv_dpcp_CPPFLAGS="-I$with_dpcp/include" - vma_cv_dpcp_LIBS="-ldpcp -lmlx5" - vma_cv_dpcp_LDFLAGS="-L$with_dpcp/lib -Wl,--rpath,$with_dpcp/lib" - if test -d "$with_dpcp/lib64"; then - vma_cv_dpcp_LDFLAGS="-L$with_dpcp/lib64 -Wl,--rpath,$with_dpcp/lib64" - fi - - CPPFLAGS="-std=c++11 $vma_cv_dpcp_CPPFLAGS $CPPFLAGS" - CXXFLAGS="-std=c++11 $CXXFLAGS" - LDFLAGS="$vma_cv_dpcp_LDFLAGS $LDFLAGS" - LIBS="$vma_cv_dpcp_LIBS $LIBS" - - AC_LANG_PUSH([C++]) - AC_CHECK_HEADER( - [mellanox/dpcp.h], - [AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include ]], - [[dpcp::provider *provider; - dpcp::provider::get_instance(provider);]])], - [vma_cv_dpcp=1]) - ]) - AC_LANG_POP() - - CPPFLAGS="$vma_cv_dpcp_save_CPPFLAGS" - CXXFLAGS="$vma_cv_dpcp_save_CXXFLAGS" - CFLAGS="$vma_cv_dpcp_save_CFLAGS" - LDFLAGS="$vma_cv_dpcp_save_LDFLAGS" - LIBS="$vma_cv_dpcp_save_LIBS" - ]) - -AC_MSG_CHECKING([for dpcp support]) -if test "$vma_cv_dpcp" -ne 0; then - CPPFLAGS="$CPPFLAGS $vma_cv_dpcp_CPPFLAGS" - LDFLAGS="$LDFLAGS $vma_cv_dpcp_LDFLAGS" - AC_SUBST([DPCP_LIBS], ["-ldpcp"]) - AC_DEFINE_UNQUOTED([DEFINED_DPCP], [1], [Define to 1 to use DPCP]) - AC_MSG_RESULT([yes]) -else - AS_IF([test "x$with_dpcp" == xno], - [AC_MSG_RESULT([no])], - [AC_MSG_ERROR([dpcp support requested but not present])]) -fi -]) diff --git a/configure.ac b/configure.ac index bce897944..25fffe7ce 100644 --- a/configure.ac +++ b/configure.ac @@ -84,7 +84,6 @@ m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) m4_include([config/m4/func.m4]) m4_include([config/m4/opt.m4]) m4_include([config/m4/verbs.m4]) -m4_include([config/m4/dpcp.m4]) m4_include([config/m4/nl.m4]) m4_include([config/m4/prof.m4]) m4_include([config/m4/compiler.m4]) @@ -216,7 +215,6 @@ AC_MSG_RESULT($ac_cv_ofed_path) VERBS_CAPABILITY_SETUP() OPT_VMA_LOGGING() PROF_IBPROF_SETUP() -DPCP_CAPABILITY_SETUP() # Enable internal performance counters # Note: uncomment setup to activate this ability diff --git a/contrib/jenkins_tests/globals.sh b/contrib/jenkins_tests/globals.sh index 548c84768..22e00d140 100755 --- a/contrib/jenkins_tests/globals.sh +++ b/contrib/jenkins_tests/globals.sh @@ -278,55 +278,6 @@ do_version_check() }' } -do_check_dpcp() -{ - local ret=0 - local version=$(echo "${jenkins_ofed}" | cut -f1-2 -d.) - - if do_version_check $version '<' '5.2' ; then - return - fi - echo "Checking dpcp usage" - - ret=0 - pushd $(pwd) > /dev/null 2>&1 - dpcp_dir=${WORKSPACE}/${prefix}/_dpcp-last - mkdir -p ${dpcp_dir} > /dev/null 2>&1 - cd ${dpcp_dir} - - set +e - if [ ! -d ${dpcp_dir}/install -a $ret -eq 0 ]; then - branch=${main:-ghprbTargetBranch} - eval "timeout -s SIGKILL 30s git clone --branch $branch git@github.com:Mellanox/dpcp.git . " > /dev/null 2>&1 - ret=$? - fi - - if [ $ret -eq 0 ]; then - last_tag=$(git describe --tags $(git rev-list --tags --max-count=1)) - if [ -z "$last_tag" ]; then - ret=1 - fi - fi - - if [ ! -d ${dpcp_dir}/install -a $ret -eq 0 ]; then - eval "git checkout $last_tag" > /dev/null 2>&1 - ret=$? - fi - - if [ ! -d ${dpcp_dir}/install -a $ret -eq 0 ]; then - eval "./autogen.sh && ./configure --prefix=${dpcp_dir}/install && make $make_opt install" > /dev/null 2>&1 - ret=$? - fi - set -e - - popd > /dev/null 2>&1 - if [ $ret -eq 0 ]; then - eval "$1=${dpcp_dir}/install" - echo "dpcp: $last_tag : ${dpcp_dir}/install" - else - echo "dpcp: no" - fi -} ####################################################### # diff --git a/contrib/test_jenkins.sh b/contrib/test_jenkins.sh index 32e87cfee..6a271dc00 100755 --- a/contrib/test_jenkins.sh +++ b/contrib/test_jenkins.sh @@ -104,15 +104,6 @@ if [ "$TARGET" == "all" -o "$TARGET" == "extra" ]; then target_list[$i]="extra: --enable-tso" i=$((i+1)) fi -if [ "$TARGET" == "all" -o "$TARGET" == "dpcp" ]; then - do_check_dpcp opt_value - if [ ! -z "${opt_value}" ]; then - target_list[$i]="dpcp: --enable-tso --enable-nginx --with-dpcp=${opt_value}" - i=$((i+1)) - else - echo "Requested dpcp support can not be executed" - fi -fi echo echo "======================================================" diff --git a/src/vma/Makefile.am b/src/vma/Makefile.am index 1fd5f4373..d2963700b 100644 --- a/src/vma/Makefile.am +++ b/src/vma/Makefile.am @@ -51,7 +51,7 @@ AM_CPPFLAGS := \ libvma_la_LDFLAGS := -no-undefined -version-number @PRJ_LIBRARY_MAJOR@:@PRJ_LIBRARY_MINOR@:@PRJ_LIBRARY_REVISION@ libvma_la_LIBADD = \ - -lrt -ldl -lpthread $(LIBNL_LIBS) $(VERBS_LIBS) $(DPCP_LIBS) \ + -lrt -ldl -lpthread $(LIBNL_LIBS) $(VERBS_LIBS) \ $(top_builddir)/src/utils/libutils.la \ $(top_builddir)/src/vlogger/libvlogger.la \ $(top_builddir)/src/state_machine/libstate_machine.la \ diff --git a/src/vma/dev/ib_ctx_handler.cpp b/src/vma/dev/ib_ctx_handler.cpp index 4c7babd96..0df33cd9d 100644 --- a/src/vma/dev/ib_ctx_handler.cpp +++ b/src/vma/dev/ib_ctx_handler.cpp @@ -72,10 +72,6 @@ ib_ctx_handler::ib_ctx_handler(struct ib_ctx_handler_desc *desc) : } m_p_ibv_context = NULL; -#ifdef DEFINED_DPCP - m_p_adapter = set_dpcp_adapter(); - if (NULL == m_p_adapter) -#endif /* DEFINED_DPCP */ { #if defined(DEFINED_ROCE_LAG) struct mlx5dv_context_attr dv_attr; @@ -135,12 +131,6 @@ ib_ctx_handler::ib_ctx_handler(struct ib_ctx_handler_desc *desc) : ibv_dealloc_pd(m_p_ibv_pd); } -#ifdef DEFINED_DPCP - if (m_p_adapter) { - delete m_p_adapter; - m_p_ibv_context = NULL; - } -#endif /* DEFINED_DPCP */ if (m_p_ibv_context) { ibv_close_device(m_p_ibv_context); m_p_ibv_context = NULL; @@ -174,12 +164,6 @@ ib_ctx_handler::~ib_ctx_handler() } delete m_p_ibv_device_attr; -#ifdef DEFINED_DPCP - if (m_p_adapter) { - delete m_p_adapter; - m_p_ibv_context = NULL; - } -#endif /* DEFINED_DPCP */ if (m_p_ibv_context) { ibv_close_device(m_p_ibv_context); m_p_ibv_context = NULL; @@ -229,113 +213,6 @@ void ib_ctx_handler::print_val() ibch_logdbg("%s", m_str); } -#ifdef DEFINED_DPCP -dpcp::adapter* ib_ctx_handler::set_dpcp_adapter() -{ - dpcp::status status = dpcp::DPCP_ERR_NO_SUPPORT; - dpcp::provider *p_provider = NULL; - dpcp::adapter_info* dpcp_lst = NULL; - size_t adapters_num = 0; - size_t i = 0; - - m_p_adapter = NULL; - if (!m_p_ibv_device) { - return NULL; - } - - status = dpcp::provider::get_instance(p_provider); - if (dpcp::DPCP_OK != status) { - ibch_logerr("failed getting provider status = %d", status); - goto err; - } - - status = p_provider->get_adapter_info_lst(NULL, adapters_num); - if (0 == adapters_num) { - ibch_logdbg("found no adapters status = %d", status); - goto err; - } - - dpcp_lst = new (std::nothrow)dpcp::adapter_info[adapters_num]; - if (!dpcp_lst) { - ibch_logerr("failed allocating memory for devices"); - goto err; - } - - status = p_provider->get_adapter_info_lst(dpcp_lst, adapters_num); - if (dpcp::DPCP_OK != status) { - ibch_logerr("failed getting adapter list"); - goto err; - } - - for (i = 0; i < adapters_num; i++) { - if (dpcp_lst[i].name == m_p_ibv_device->name) { - dpcp::adapter *adapter = NULL; - - status = p_provider->open_adapter(dpcp_lst[i].name, adapter); - if ((dpcp::DPCP_OK == status) && (adapter)) { - int ret = 0; - struct ibv_context *ctx = NULL; - struct ibv_pd *pd = NULL; - mlx5dv_obj mlx5_obj; - - ctx = (ibv_context*)adapter->get_ibv_context(); - if (!ctx) { - ibch_logerr("failed getting context for adapter %p (errno=%d %m) ", - adapter, errno); - delete adapter; - goto err; - } - - pd = ibv_alloc_pd(ctx); - if (!pd) { - ibch_logerr("failed pd allocation for %p context (errno=%d %m) ", - ctx, errno); - delete adapter; - goto err; - } - - mlx5_obj.pd.in = pd; - mlx5dv_pd out_pd; - mlx5_obj.pd.out = &out_pd; - - ret = vma_ib_mlx5dv_init_obj(&mlx5_obj, MLX5DV_OBJ_PD); - if (ret) { - ibch_logerr("failed getting mlx5_pd for %p (errno=%d %m) ", - m_p_ibv_pd, errno); - ibv_dealloc_pd(pd); - delete adapter; - goto err; - } - - adapter->set_pd(out_pd.pdn, pd); - status = adapter->open(); - if (dpcp::DPCP_OK != status) { - ibch_logerr("failed opening dpcp adapter %s got %d", - adapter->get_name().c_str(), status); - ibv_dealloc_pd(pd); - delete adapter; - goto err; - } - - m_p_adapter = adapter; - m_p_ibv_context = ctx; - m_p_ibv_pd = pd; - ibch_logdbg("dpcp adapter: %s is up", adapter->get_name().c_str()); - } - - break; - } - } - -err: - if (dpcp_lst) { - delete[] dpcp_lst; - } - - return m_p_adapter; -} -#endif /* DEFINED_DPCP */ - void ib_ctx_handler::set_ctx_time_converter_status(ts_conversion_mode_t conversion_mode) { if (m_p_ctx_time_converter != NULL) { diff --git a/src/vma/dev/ib_ctx_handler.h b/src/vma/dev/ib_ctx_handler.h index 5c4c5abd0..52f4222f6 100644 --- a/src/vma/dev/ib_ctx_handler.h +++ b/src/vma/dev/ib_ctx_handler.h @@ -42,10 +42,6 @@ #include "vma/ib/base/verbs_extra.h" #include "utils/lock_wrapper.h" -#ifdef DEFINED_DPCP -#include -#endif /* DEFINED_DPCP */ - typedef std::unordered_map mr_map_lkey_t; @@ -77,10 +73,6 @@ class ib_ctx_handler : public event_handler_ibverbs ibv_device* get_ibv_device() { return m_p_ibv_device; } inline char* get_ibname() { return (m_p_ibv_device ? m_p_ibv_device->name : (char *)""); } struct ibv_context* get_ibv_context() { return m_p_ibv_context; } -#ifdef DEFINED_DPCP - dpcp::adapter* set_dpcp_adapter(); - dpcp::adapter* get_dpcp_adapter() { return m_p_adapter; } -#endif /* DEFINED_DPCP */ vma_ibv_device_attr* get_ibv_device_attr() { return vma_get_device_orig_attr(m_p_ibv_device_attr); } #ifdef DEFINED_TSO vma_ibv_device_attr_ex* get_ibv_device_attr_ex() { return m_p_ibv_device_attr; } @@ -113,9 +105,6 @@ class ib_ctx_handler : public event_handler_ibverbs void handle_event_device_fatal(); ibv_device* m_p_ibv_device; // HCA handle struct ibv_context* m_p_ibv_context; -#ifdef DEFINED_DPCP - dpcp::adapter *m_p_adapter; -#endif /* DEFINED_DPCP */ vma_ibv_device_attr_ex* m_p_ibv_device_attr; ibv_pd* m_p_ibv_pd; bool m_flow_tag_enabled; diff --git a/src/vma/sock/sock-redirect.cpp b/src/vma/sock/sock-redirect.cpp index 9b232e609..6002e1560 100644 --- a/src/vma/sock/sock-redirect.cpp +++ b/src/vma/sock/sock-redirect.cpp @@ -756,47 +756,6 @@ int vma_dereg_mr_on_ring(int __fd, void *addr, size_t length) } } -extern "C" -int vma_get_dpcp_devices(uintptr_t **devices, size_t *devices_num) -{ -#ifdef DEFINED_DPCP - ib_context_map_t *ib_ctx_map = NULL; - ib_ctx_handler *p_ib_ctx_h = NULL; - size_t found_devs = 0; - - if (!devices_num) { - return EINVAL; - } - - ib_ctx_map = g_p_ib_ctx_handler_collection->get_ib_cxt_list(); - if (ib_ctx_map) { - ib_context_map_t::iterator iter; - - for (iter = ib_ctx_map->begin(); iter != ib_ctx_map->end(); iter++) { - p_ib_ctx_h = iter->second; - if (p_ib_ctx_h->get_dpcp_adapter()) { - if (devices && (found_devs < *devices_num)) { - devices[found_devs] = (uintptr_t*)p_ib_ctx_h->get_dpcp_adapter(); - } - found_devs++; - } - } - } - - *devices_num = found_devs; - srdr_logdbg_entry("returned %zd devices", found_devs); - - return 0; -#else - NOT_IN_USE(devices); - NOT_IN_USE(devices_num); - VLOG_PRINTF_ONCE_THEN_ALWAYS(VLOG_WARNING, VLOG_DEBUG, - "vma_get_dpcp_devices is no supported"); - errno = EOPNOTSUPP; - return -1; -#endif /* DEFINED_DPCP */ -} - //----------------------------------------------------------------------------- // replacement functions //----------------------------------------------------------------------------- @@ -1133,7 +1092,6 @@ int getsockopt(int __fd, int __level, int __optname, SET_EXTRA_API(socketxtreme_free_vma_buff, enable_socketxtreme ? vma_socketxtreme_free_vma_buff : dummy_vma_socketxtreme_free_vma_buff, VMA_EXTRA_API_SOCKETXTREME_FREE_VMA_BUFF); SET_EXTRA_API(dump_fd_stats, vma_dump_fd_stats, VMA_EXTRA_API_DUMP_FD_STATS); SET_EXTRA_API(vma_modify_ring, vma_modify_ring, VMA_EXTRA_API_MODIFY_RING); - SET_EXTRA_API(get_dpcp_devices, vma_get_dpcp_devices, VMA_EXTRA_API_GET_DPCP_DEVICES); *((vma_api_t**)__optval) = vma_api; return 0; } diff --git a/src/vma/vma_extra.h b/src/vma/vma_extra.h index c84ef8980..3c59c7721 100644 --- a/src/vma/vma_extra.h +++ b/src/vma/vma_extra.h @@ -366,7 +366,6 @@ typedef enum { VMA_EXTRA_API_REGISTER_MEMORY_ON_RING = (1 << 17), VMA_EXTRA_API_DEREGISTER_MEMORY_ON_RING = (1 << 18), VMA_EXTRA_API_MODIFY_RING = (1 << 20), - VMA_EXTRA_API_GET_DPCP_DEVICES = (1 << 21) } vma_extra_api_mask; /** @@ -715,23 +714,6 @@ struct __attribute__ ((packed)) vma_api_t { */ uint64_t vma_extra_supported_mask; - /** - * get dpcp devices allocated by VMA - * - * @param devices - pointer to write the data to. can be NULL see notes - * @param devices_num - IN\OUT parameter - * IN - devices size given by user - * OUT- devices returned to user - * - * @return 0 on success -1 otherwise - * - * @note application can: - * call twice once with devices == NULL and get the size needed to allocate - * and call again to get the filled device array. - * if application is called with big enough buffer VMA will update the - * size actually used. - */ - int (*get_dpcp_devices)(uintptr_t **devices, size_t *devices_num); }; /**