diff --git a/.github/workflows/bullseye-coverage.yml b/.github/workflows/bullseye-coverage.yml index a7d9d97ab35..a4139fca908 100644 --- a/.github/workflows/bullseye-coverage.yml +++ b/.github/workflows/bullseye-coverage.yml @@ -563,13 +563,11 @@ jobs: STAGE_TAGS+=",provider" if [[ '${{ matrix.stage }}' = *\ Verbs\ * ]]; then FTEST_ARG+=' --provider ofi+verbs' - INST_RPMS+=' mercury-libfabric' elif [[ '${{ matrix.stage }}' = *\ UCX\ * ]]; then FTEST_ARG+=' --provider ucx+dc_x' INST_RPMS+=' mercury-ucx' elif [[ '${{ matrix.stage }}' = *\ TCP\ * ]]; then FTEST_ARG+=' --provider ofi+tcp' - INST_RPMS+=' mercury-libfabric' else echo 'Unknown provider in ${{ matrix.stage }}' exit 1 diff --git a/.github/workflows/rpm-build-and-test.yml b/.github/workflows/rpm-build-and-test.yml index 4bbb942d788..2483fbbaa7b 100644 --- a/.github/workflows/rpm-build-and-test.yml +++ b/.github/workflows/rpm-build-and-test.yml @@ -572,13 +572,11 @@ jobs: STAGE_TAGS+=",provider" if [[ '${{ matrix.stage }}' = *\ Verbs\ * ]]; then FTEST_ARG+=' --provider ofi+verbs' - INST_RPMS+=' mercury-libfabric' elif [[ '${{ matrix.stage }}' = *\ UCX\ * ]]; then FTEST_ARG+=' --provider ucx+dc_x' INST_RPMS+=' mercury-ucx' elif [[ '${{ matrix.stage }}' = *\ TCP\ * ]]; then FTEST_ARG+=' --provider ofi+tcp' - INST_RPMS+=' mercury-libfabric' else echo 'Unknown provider in ${{ matrix.stage }}' exit 1 diff --git a/Jenkinsfile b/Jenkinsfile index 26ce0473a71..98dda0fcbfe 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1048,8 +1048,7 @@ pipeline { steps { job_step_update( testRpm(inst_repos: daosRepos(), - daos_pkg_version: daosPackagesVersion(next_version()), - inst_rpms: 'mercury-libfabric') + daos_pkg_version: daosPackagesVersion(next_version())) ) } post { @@ -1101,8 +1100,7 @@ pipeline { } */ job_step_update( testRpm(inst_repos: daosRepos(), - daos_pkg_version: daosPackagesVersion(next_version()), - inst_rpms: 'mercury-libfabric') + daos_pkg_version: daosPackagesVersion(next_version())) ) } post { diff --git a/ci/provisioning/post_provision_config_common.sh b/ci/provisioning/post_provision_config_common.sh index 3cac657efaf..257e6dfe2d9 100755 --- a/ci/provisioning/post_provision_config_common.sh +++ b/ci/provisioning/post_provision_config_common.sh @@ -1,7 +1,7 @@ #!/bin/bash # # Copyright 2021-2023 Intel Corporation. -# Copyright 2025-2026 Hewlett Packard Enterprise Development LP +# Copyright 2025 Hewlett Packard Enterprise Development LP # # SPDX-License-Identifier: BSD-2-Clause-Patent # @@ -32,7 +32,7 @@ fi # shellcheck disable=SC1091 . /etc/os-release # shellcheck disable=SC2034 -EXCLUDE_UPGRADE=mercury,mercury-\*,daos,daos-\* +EXCLUDE_UPGRADE=mercury,daos,daos-\* if rpm -qa | grep mlnx; then # packages not to allow upgrading if MLNX OFED is installed EXCLUDE_UPGRADE+=,openmpi,\*mlnx\*,\*ucx\* diff --git a/ci/unit/required_packages.sh b/ci/unit/required_packages.sh index 5a57b0cb054..bbd3313155a 100755 --- a/ci/unit/required_packages.sh +++ b/ci/unit/required_packages.sh @@ -1,10 +1,5 @@ #!/bin/bash -# -# (C) Copyright 2025 Google LLC -# Copyright 2025-2026 Hewlett Packard Enterprise Development LP -# -# SPDX-License-Identifier: BSD-2-Clause-Patent -# + set -eu # No longer used but provided by pipeline-lib @@ -29,7 +24,6 @@ pkgs="$(utils/rpms/package_version.sh argobots lib) \ $(utils/rpms/package_version.sh libfabric debug) \ $(utils/rpms/package_version.sh mercury dev) \ $(utils/rpms/package_version.sh mercury debug) \ - $(utils/rpms/package_version.sh mercury lib mercury_libfabric) \ $(utils/rpms/package_version.sh pmdk lib pmemobj) \ $(utils/rpms/package_version.sh pmdk debug pmemobj) \ $(utils/rpms/package_version.sh pmdk debug pmem) \ diff --git a/deps/patches/mercury/0001_dep_versions.patch b/deps/patches/mercury/0001_dep_versions.patch deleted file mode 100644 index 6b1d6a71a80..00000000000 --- a/deps/patches/mercury/0001_dep_versions.patch +++ /dev/null @@ -1,74 +0,0 @@ -From 0a7756d4ef2f329fa7caa8e4052a099a91816f2f Mon Sep 17 00:00:00 2001 -From: Jerome Soumagne -Date: Mon, 24 Nov 2025 16:44:14 -0600 -Subject: [PATCH 1/3] NA OFI/UCX: print version infos - -NA OFI: fix log warning ---- - src/na/na_ofi.c | 12 +++++++----- - src/na/na_ucx.c | 10 ++++++++++ - 2 files changed, 17 insertions(+), 5 deletions(-) - -diff --git a/src/na/na_ofi.c b/src/na/na_ofi.c -index c7c3e0b3..682efe65 100644 ---- a/src/na/na_ofi.c -+++ b/src/na/na_ofi.c -@@ -8048,8 +8048,10 @@ na_ofi_check_protocol(const char *protocol_name) - uint32_t runtime_version = fi_version(); - na_return_t na_ret; - -- NA_LOG_SUBSYS_DEBUG(cls, "Querying info on libfabric v%d.%d", -- FI_MAJOR(runtime_version), FI_MINOR(runtime_version)); -+ NA_LOG_SUBSYS_INFO(cls, -+ "Querying info on libfabric (runtime v%d.%d, API v%d.%d)", -+ FI_MAJOR(runtime_version), FI_MINOR(runtime_version), -+ FI_MAJOR(FI_COMPILE_VERSION), FI_MINOR(FI_COMPILE_VERSION)); - NA_CHECK_SUBSYS_ERROR(cls, FI_VERSION_LT(runtime_version, NA_OFI_VERSION), - out, accept, false, - "runtime libfabric version (v%d.%d) is lower than required version " -@@ -9105,7 +9107,7 @@ na_ofi_mem_handle_create(na_class_t NA_UNUSED *na_class, void *buf, - - NA_LOG_SUBSYS_DEBUG(mem, - "Created mem handle %p (iov_base=%p, iov_len=%zu, iovcnt=1, " -- "flags=0x%lx, len=%zu)", -+ "flags=0x%" PRIx8 ", len=%" PRIu64 ")", - (void *) na_ofi_mem_handle, na_ofi_mem_handle->desc.iov.s[0].iov_base, - na_ofi_mem_handle->desc.iov.s[0].iov_len, - na_ofi_mem_handle->desc.info.flags, na_ofi_mem_handle->desc.info.len); -@@ -9444,8 +9446,8 @@ na_ofi_mem_handle_deserialize(na_class_t NA_UNUSED *na_class, - na_ofi_mem_handle->desc.info.iovcnt); - - NA_LOG_SUBSYS_DEBUG(mem, -- "Deserialized mem handle %p (iov_base=%p, iov_len=%zu, iovcnt=%zu, " -- "flags=0x%lx, len=%zu)", -+ "Deserialized mem handle %p (iov_base=%p, iov_len=%zu, iovcnt=%" PRIu64 -+ ", flags=0x%" PRIx8 ", len=%" PRIu64 ")", - (void *) na_ofi_mem_handle, na_ofi_mem_handle->desc.iov.s[0].iov_base, - na_ofi_mem_handle->desc.iov.s[0].iov_len, - na_ofi_mem_handle->desc.info.iovcnt, na_ofi_mem_handle->desc.info.flags, -diff --git a/src/na/na_ucx.c b/src/na/na_ucx.c -index 96501b27..7c0ac8d4 100644 ---- a/src/na/na_ucx.c -+++ b/src/na/na_ucx.c -@@ -3433,8 +3433,18 @@ na_ucx_check_protocol(const char *protocol_name) - .field_mask = UCP_PARAM_FIELD_FEATURES, .features = NA_UCX_FEATURES}; - ucp_context_h context = NULL; - ucs_status_t status; -+ unsigned int runtime_major_version, runtime_minor_version, -+ runtime_patch_version; - bool accept = false; - -+ ucp_get_version( -+ &runtime_major_version, &runtime_minor_version, &runtime_patch_version); -+ -+ NA_LOG_SUBSYS_INFO(cls, -+ "Querying info on UCX (runtime v%u.%u.%u, API v%u.%u)", -+ runtime_major_version, runtime_minor_version, runtime_patch_version, -+ UCP_API_MAJOR, UCP_API_MINOR); -+ - status = ucp_config_read(NULL, NULL, &config); - NA_CHECK_SUBSYS_ERROR_NORET(cls, status != UCS_OK, done, - "ucp_config_read() failed (%s)", ucs_status_string(status)); --- -2.52.0 - diff --git a/deps/patches/mercury/0001_na_ucx.patch b/deps/patches/mercury/0001_na_ucx.patch new file mode 100644 index 00000000000..57b39feef9d --- /dev/null +++ b/deps/patches/mercury/0001_na_ucx.patch @@ -0,0 +1,110 @@ +diff --git a/src/na/na_ucx.c b/src/na/na_ucx.c +index 84eb8b0..e4b6676 100644 +--- a/src/na/na_ucx.c ++++ b/src/na/na_ucx.c +@@ -614,7 +614,7 @@ na_ucx_addr_map_update(struct na_ucx_class *na_ucx_class, + */ + static na_return_t + na_ucx_addr_map_remove( +- struct na_ucx_map *na_ucx_map, ucs_sock_addr_t *addr_key); ++ struct na_ucx_map *na_ucx_map, struct na_ucx_addr *remove_addr); + + /** + * Hash connection ID. +@@ -1688,8 +1688,12 @@ na_ucp_listener_conn_cb(ucp_conn_request_h conn_request, void *arg) + .addr = (const struct sockaddr *) &conn_request_attrs.client_address, + .addrlen = sizeof(conn_request_attrs.client_address)}; + na_ucx_addr = na_ucx_addr_map_lookup(&na_ucx_class->addr_map, &addr_key); +- NA_CHECK_SUBSYS_ERROR_NORET(addr, na_ucx_addr != NULL, error, +- "An entry is already present for this address"); ++ ++ if (na_ucx_addr != NULL) { ++ NA_LOG_SUBSYS_WARNING(addr, ++ "An entry is already present for this address"); ++ na_ucx_addr_map_remove(&na_ucx_class->addr_map, na_ucx_addr); ++ } + + /* Insert new entry and create new address */ + na_ret = na_ucx_addr_map_insert(na_ucx_class, &na_ucx_class->addr_map, +@@ -1937,10 +1941,14 @@ na_ucp_ep_error_cb( + static void + na_ucp_ep_close(ucp_ep_h ep) + { +- ucs_status_ptr_t status_ptr = ucp_ep_close_nb(ep, UCP_EP_CLOSE_MODE_FORCE); ++ const ucp_request_param_t close_params = { ++ .op_attr_mask = UCP_OP_ATTR_FIELD_FLAGS, ++ .flags = UCP_EP_CLOSE_FLAG_FORCE}; ++ ucs_status_ptr_t status_ptr = ucp_ep_close_nbx(ep, &close_params); ++ + NA_CHECK_SUBSYS_ERROR_DONE(addr, + status_ptr != NULL && UCS_PTR_IS_ERR(status_ptr), +- "ucp_ep_close_nb() failed (%s)", ++ "ucp_ep_close_nbx() failed (%s)", + ucs_status_string(UCS_PTR_STATUS(status_ptr))); + } + +@@ -2722,7 +2730,7 @@ unlock: + + /*---------------------------------------------------------------------------*/ + static na_return_t +-na_ucx_addr_map_remove(struct na_ucx_map *na_ucx_map, ucs_sock_addr_t *addr_key) ++na_ucx_addr_map_remove(struct na_ucx_map *na_ucx_map, struct na_ucx_addr *remove_addr) + { + struct na_ucx_addr *na_ucx_addr = NULL; + na_return_t ret = NA_SUCCESS; +@@ -2731,13 +2739,14 @@ na_ucx_addr_map_remove(struct na_ucx_map *na_ucx_map, ucs_sock_addr_t *addr_key) + hg_thread_rwlock_wrlock(&na_ucx_map->lock); + + na_ucx_addr = hg_hash_table_lookup( +- na_ucx_map->key_map, (hg_hash_table_key_t) addr_key); +- if (na_ucx_addr == HG_HASH_TABLE_NULL) ++ na_ucx_map->key_map, (hg_hash_table_key_t) &remove_addr->addr_key); ++ ++ if (na_ucx_addr == HG_HASH_TABLE_NULL || na_ucx_addr->ucp_ep != remove_addr->ucp_ep) + goto unlock; + + /* Remove addr key from primary map */ + rc = hg_hash_table_remove( +- na_ucx_map->key_map, (hg_hash_table_key_t) addr_key); ++ na_ucx_map->key_map, (hg_hash_table_key_t) &na_ucx_addr->addr_key); + NA_CHECK_SUBSYS_ERROR(addr, rc != 1, unlock, ret, NA_NOENTRY, + "hg_hash_table_remove() failed"); + +@@ -2841,7 +2850,7 @@ na_ucx_addr_release(struct na_ucx_addr *na_ucx_addr) + NA_UCX_PRINT_ADDR_KEY_INFO("Removing address", &na_ucx_addr->addr_key); + + na_ucx_addr_map_remove( +- &na_ucx_addr->na_ucx_class->addr_map, &na_ucx_addr->addr_key); ++ &na_ucx_addr->na_ucx_class->addr_map, na_ucx_addr); + } + + if (na_ucx_addr->ucp_ep != NULL) { +@@ -3023,6 +3032,18 @@ na_ucx_rma(struct na_ucx_class NA_UNUSED *na_ucx_class, na_context_t *context, + + /* There is no need to have a fully resolved address to start an RMA. + * This is only necessary for two-sided communication. */ ++ /* The above assumption is now in question, so the following will resolve ++ * the address if required. */ ++ ++ /* Check addr to ensure the EP for that addr is still valid */ ++ if (!(hg_atomic_get32(&na_ucx_addr->status) & NA_UCX_ADDR_RESOLVED)) { ++ ret = na_ucx_addr_map_update( ++ na_ucx_class, &na_ucx_class->addr_map, na_ucx_addr); ++ NA_CHECK_SUBSYS_NA_ERROR( ++ addr, error, ret, "Could not update NA UCX address"); ++ } ++ NA_CHECK_SUBSYS_ERROR(msg, na_ucx_addr->ucp_ep == NULL, error, ret, ++ NA_ADDRNOTAVAIL, "UCP endpoint is NULL for that address"); + + /* TODO UCX requires the remote key to be bound to the origin, do we need a + * new API? */ +@@ -3061,6 +3082,9 @@ na_ucx_rma_key_resolve(ucp_ep_h ep, struct na_ucx_mem_handle *na_ucx_mem_handle, + + hg_thread_mutex_lock(&na_ucx_mem_handle->rkey_unpack_lock); + ++ NA_CHECK_SUBSYS_ERROR( ++ mem, ep == NULL, error, ret, NA_INVALID_ARG, "Invalid endpoint (%p)", ep); ++ + switch (hg_atomic_get32(&na_ucx_mem_handle->type)) { + case NA_UCX_MEM_HANDLE_REMOTE_PACKED: { + ucs_status_t status = ucp_ep_rkey_unpack(ep, diff --git a/deps/patches/mercury/0002_na_ucx_ep_flush.patch b/deps/patches/mercury/0002_na_ucx_ep_flush.patch new file mode 100644 index 00000000000..f7b38d304aa --- /dev/null +++ b/deps/patches/mercury/0002_na_ucx_ep_flush.patch @@ -0,0 +1,64 @@ +diff --git a/src/na/na_ucx.c b/src/na/na_ucx.c +index 6e9c3b0..2f157da 100644 +--- a/src/na/na_ucx.c ++++ b/src/na/na_ucx.c +@@ -441,6 +441,12 @@ na_ucp_ep_create(ucp_worker_h worker, ucp_ep_params_t *ep_params, + static void + na_ucp_ep_error_cb(void *arg, ucp_ep_h ep, ucs_status_t status); + ++/** ++ * Flush endpoint. ++ */ ++static ucs_status_ptr_t ++na_ucp_ep_flush(ucp_ep_h ep); ++ + /** + * Close endpoint. + */ +@@ -1940,6 +1946,21 @@ na_ucp_ep_error_cb( + na_ucx_addr_ref_decr(na_ucx_addr); + } + ++/*---------------------------------------------------------------------------*/ ++static ucs_status_ptr_t ++na_ucp_ep_flush(ucp_ep_h ep) ++{ ++ const ucp_request_param_t flush_params = { ++ .op_attr_mask = 0}; ++ ucs_status_ptr_t status_ptr = ucp_ep_flush_nbx(ep, &flush_params); ++ ++ NA_CHECK_SUBSYS_ERROR_DONE(addr, ++ status_ptr != NULL && UCS_PTR_IS_ERR(status_ptr), ++ "ucp_ep_flush_nb() failed (%s)", ++ ucs_status_string(UCS_PTR_STATUS(status_ptr))); ++ return status_ptr; ++} ++ + /*---------------------------------------------------------------------------*/ + static void + na_ucp_ep_close(ucp_ep_h ep) +@@ -2859,8 +2880,23 @@ na_ucx_addr_release(struct na_ucx_addr *na_ucx_addr) + if (na_ucx_addr->ucp_ep != NULL) { + /* NB. for deserialized addresses that are not "connected" addresses, do + * not close the EP */ +- if (na_ucx_addr->worker_addr == NULL) ++ if (na_ucx_addr->worker_addr == NULL) { ++ if (!na_ucx_addr->na_ucx_class->ucp_listener) { ++ ucs_status_ptr_t status_ptr = na_ucp_ep_flush(na_ucx_addr->ucp_ep); ++ ++ if (UCS_PTR_IS_PTR(status_ptr)) { ++ ucs_status_t status; ++ ++ do { ++ ucp_worker_progress(na_ucx_addr->na_ucx_class->ucp_worker); ++ status = ucp_request_check_status(status_ptr); ++ } while (status == UCS_INPROGRESS); ++ ucp_request_free(status_ptr); ++ } ++ } ++ + na_ucp_ep_close(na_ucx_addr->ucp_ep); ++ } + na_ucx_addr->ucp_ep = NULL; + } + diff --git a/deps/patches/mercury/0002_ofi_counters.patch b/deps/patches/mercury/0002_ofi_counters.patch deleted file mode 100644 index 7cbdc543bd8..00000000000 --- a/deps/patches/mercury/0002_ofi_counters.patch +++ /dev/null @@ -1,1196 +0,0 @@ -From 187759c6cd99205c52fc5b77fdb6c2d3536f9021 Mon Sep 17 00:00:00 2001 -From: Jerome Soumagne -Date: Thu, 18 Dec 2025 17:47:03 -0600 -Subject: [PATCH 2/3] NA OFI: add counters for monitoring tx/rx/rma/cq counts - -Monitor mr and addr counts - -NA: add NA_Diag_dump_counters() routine to dump counters -if HG_LOG_LEVEL>=min_debug is set - -HG Core: clean up counters - -HG util: add ability to remove counters - -NA OFI: finalize counters after fabric is closed - -HG util: bump minor version ---- - CMakeLists.txt | 2 +- - src/mercury.c | 1 + - src/mercury_core.c | 40 ++--- - src/na/CMakeLists.txt | 5 + - src/na/na.c | 9 + - src/na/na.h | 6 + - src/na/na_config.h.in | 1 + - src/na/na_ofi.c | 360 +++++++++++++++++++++++++++++++++------- - src/util/mercury_dlog.c | 37 ++++- - src/util/mercury_dlog.h | 18 ++ - src/util/mercury_log.c | 14 +- - src/util/mercury_log.h | 8 + - src/util/version.txt | 2 +- - 13 files changed, 413 insertions(+), 90 deletions(-) - -diff --git a/CMakeLists.txt b/CMakeLists.txt -index e4e79711..e71944f1 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -192,7 +192,7 @@ endif() - # Enable diagnostics counters separately from debug. - #------------------------------------------------------------------------------ - option(MERCURY_ENABLE_COUNTERS "Enable diagnostics counters." OFF) --if(MERCURY_ENABLE_COUNTERS) -+if(MERCURY_ENABLE_COUNTERS AND NOT WIN32) - set(HG_HAS_DIAG 1) - else() - set(HG_HAS_DIAG 0) -diff --git a/src/mercury.c b/src/mercury.c -index 2c062384..6f0376ee 100644 ---- a/src/mercury.c -+++ b/src/mercury.c -@@ -1178,6 +1178,7 @@ HG_Diag_dump_counters(void) - #ifndef _WIN32 - hg_log_dump_counters(&HG_LOG_OUTLET(hg_diag)); - #endif -+ NA_Diag_dump_counters(); - } - - /*---------------------------------------------------------------------------*/ -diff --git a/src/mercury_core.c b/src/mercury_core.c -index 97cc4fb2..7abd8a47 100644 ---- a/src/mercury_core.c -+++ b/src/mercury_core.c -@@ -185,7 +185,7 @@ struct hg_core_private_class { - struct hg_core_map rpc_map; /* RPC Map */ - struct hg_core_more_data_cb more_data_cb; /* More data callbacks */ - na_tag_t request_max_tag; /* Max value for tag */ --#if defined(HG_HAS_DIAG) && !defined(_WIN32) -+#ifdef HG_HAS_DIAG - struct hg_core_counters counters; /* Diag counters */ - #endif - hg_atomic_int32_t n_contexts; /* Total number of contexts */ -@@ -369,7 +369,7 @@ struct hg_core_private_handle { - uint8_t cookie; /* Cookie */ - bool multi_recv_copy; /* Copy on multi-recv */ - bool reuse; /* Re-use handle once ref_count is 0 */ --#if defined(HG_HAS_DIAG) && !defined(_WIN32) -+#ifdef HG_HAS_DIAG - bool active; - #endif - }; -@@ -405,7 +405,7 @@ hg_core_op_type_to_string(enum hg_core_op_type op_type); - /** - * Init counters. - */ --#if defined(HG_HAS_DIAG) && !defined(_WIN32) -+#ifdef HG_HAS_DIAG - static void - hg_core_counters_init(struct hg_core_counters *hg_core_counters); - #endif -@@ -447,7 +447,7 @@ hg_core_finalize(struct hg_core_private_class *hg_core_class); - /** - * Get counters. - */ --#if defined(HG_HAS_DIAG) && !defined(_WIN32) -+#ifdef HG_HAS_DIAG - static void - hg_core_class_get_counters(const struct hg_core_counters *counters, - struct hg_diag_counters *diag_counters); -@@ -1091,7 +1091,7 @@ hg_core_op_type_to_string(enum hg_core_op_type op_type) - #endif - - /*---------------------------------------------------------------------------*/ --#if defined(HG_HAS_DIAG) && !defined(_WIN32) -+#ifdef HG_HAS_DIAG - static void - hg_core_counters_init(struct hg_core_counters *hg_core_counters) - { -@@ -1325,7 +1325,7 @@ hg_core_init(const char *na_info_string, bool na_listen, unsigned int version, - hg_core_class->init_info.listen = na_listen; - - /* Stats / counters */ --#if defined(HG_HAS_DIAG) && !defined(_WIN32) -+#ifdef HG_HAS_DIAG - hg_core_counters_init(&hg_core_class->counters); - #endif - -@@ -1521,7 +1521,7 @@ error: - } - - /*---------------------------------------------------------------------------*/ --#if defined(HG_HAS_DIAG) && !defined(_WIN32) -+#ifdef HG_HAS_DIAG - static void - hg_core_class_get_counters(const struct hg_core_counters *counters, - struct hg_diag_counters *diag_counters) -@@ -3445,7 +3445,7 @@ hg_core_destroy(struct hg_core_private_handle *hg_core_handle) - return HG_SUCCESS; /* Cannot free yet */ - } - --#if defined(HG_HAS_DIAG) && !defined(_WIN32) -+#ifdef HG_HAS_DIAG - if (hg_core_handle->active) { - hg_atomic_decr64(HG_CORE_HANDLE_CLASS(hg_core_handle) - ->counters.rpc_req_recv_active_count); -@@ -4048,7 +4048,7 @@ hg_core_forward(struct hg_core_private_handle *hg_core_handle, - hg_core_handle->request_callback = callback; - hg_core_handle->request_arg = arg; - --#if defined(HG_HAS_DIAG) && !defined(_WIN32) -+#ifdef HG_HAS_DIAG - /* Increment counter */ - hg_atomic_incr64( - HG_CORE_HANDLE_CLASS(hg_core_handle)->counters.rpc_req_sent_count); -@@ -4263,7 +4263,7 @@ hg_core_respond(struct hg_core_private_handle *hg_core_handle, - hg_core_handle->response_callback = callback; - hg_core_handle->response_arg = arg; - --#if defined(HG_HAS_DIAG) && !defined(_WIN32) -+#ifdef HG_HAS_DIAG - /* Increment counter */ - hg_atomic_incr64( - HG_CORE_HANDLE_CLASS(hg_core_handle)->counters.rpc_resp_sent_count); -@@ -4499,7 +4499,7 @@ hg_core_recv_input_cb(const struct na_cb_info *callback_info) - hg_thread_spin_lock(&hg_core_handle_pool->pending_list.lock); - LIST_REMOVE(hg_core_handle, pending); - hg_thread_spin_unlock(&hg_core_handle_pool->pending_list.lock); --#if defined(HG_HAS_DIAG) && !defined(_WIN32) -+#ifdef HG_HAS_DIAG - /* Increment counter */ - hg_atomic_incr64(HG_CORE_HANDLE_CLASS(hg_core_handle) - ->counters.rpc_req_recv_active_count); -@@ -4608,7 +4608,7 @@ hg_core_multi_recv_input_cb(const struct na_cb_info *callback_info) - ret = hg_core_handle_pool_get(context->handle_pool, &hg_core_handle); - HG_CHECK_SUBSYS_HG_ERROR( - rpc, error, ret, "Could not get handle from pool"); --#if defined(HG_HAS_DIAG) && !defined(_WIN32) -+#ifdef HG_HAS_DIAG - /* Increment counter */ - hg_atomic_incr64(HG_CORE_HANDLE_CLASS(hg_core_handle) - ->counters.rpc_req_recv_active_count); -@@ -4665,7 +4665,7 @@ hg_core_multi_recv_input_cb(const struct na_cb_info *callback_info) - "Copying multi-recv payload of size %zu for handle (%p)", - hg_core_handle->core_handle.in_buf_used, - (void *) hg_core_handle); --#if defined(HG_HAS_DIAG) && !defined(_WIN32) -+#ifdef HG_HAS_DIAG - /* Increment counter */ - hg_atomic_incr64(HG_CORE_CONTEXT_CLASS(context) - ->counters.rpc_multi_recv_copy_count); -@@ -4763,7 +4763,7 @@ hg_core_process_input(struct hg_core_private_handle *hg_core_handle) - uint32_t flags = (uint32_t) hg_atomic_get32(&hg_core_handle->flags); - hg_return_t ret; - --#if defined(HG_HAS_DIAG) && !defined(_WIN32) -+#ifdef HG_HAS_DIAG - /* Increment counter */ - hg_atomic_incr64(hg_core_class->counters.rpc_req_recv_count); - #endif -@@ -4812,7 +4812,7 @@ hg_core_process_input(struct hg_core_private_handle *hg_core_handle) - "Handle (%p) expected_count incr to %" PRId32, - (void *) hg_core_handle, expected_count); - --#if defined(HG_HAS_DIAG) && !defined(_WIN32) -+#ifdef HG_HAS_DIAG - /* Increment counter */ - hg_atomic_incr64(hg_core_class->counters.rpc_req_extra_count); - #endif -@@ -4936,7 +4936,7 @@ hg_core_process_output(struct hg_core_private_handle *hg_core_handle) - uint32_t flags = (uint32_t) hg_atomic_get32(&hg_core_handle->flags); - hg_return_t ret; - --#if defined(HG_HAS_DIAG) && !defined(_WIN32) -+#ifdef HG_HAS_DIAG - /* Increment counter */ - hg_atomic_incr64(hg_core_class->counters.rpc_resp_recv_count); - #endif -@@ -4980,7 +4980,7 @@ hg_core_process_output(struct hg_core_private_handle *hg_core_handle) - "Handle (%p) expected_count incr to %" PRId32, - (void *) hg_core_handle, expected_count); - --#if defined(HG_HAS_DIAG) && !defined(_WIN32) -+#ifdef HG_HAS_DIAG - /* Increment counter */ - hg_atomic_incr64(hg_core_class->counters.rpc_resp_extra_count); - #endif -@@ -5319,7 +5319,7 @@ hg_core_completion_add(struct hg_core_context *core_context, - struct hg_core_completion_queue *backfill_queue = &context->backfill_queue; - int rc; - --#if defined(HG_HAS_DIAG) && !defined(_WIN32) -+#ifdef HG_HAS_DIAG - /* Increment counter */ - if (hg_completion_entry->op_type == HG_BULK) - hg_atomic_incr64(HG_CORE_CONTEXT_CLASS(context)->counters.bulk_count); -@@ -6212,7 +6212,7 @@ hg_return_t - HG_Core_class_get_counters(const hg_core_class_t *hg_core_class, - struct hg_diag_counters *diag_counters) - { --#if defined(HG_HAS_DIAG) && !defined(_WIN32) -+#ifdef HG_HAS_DIAG - const struct hg_core_private_class *private_class = - (const struct hg_core_private_class *) hg_core_class; - #endif -@@ -6222,7 +6222,7 @@ HG_Core_class_get_counters(const hg_core_class_t *hg_core_class, - HG_INVALID_ARG, "NULL HG core class"); - HG_CHECK_SUBSYS_ERROR(cls, diag_counters == NULL, error, ret, - HG_INVALID_ARG, "NULL pointer to diag_counters"); --#if defined(HG_HAS_DIAG) && !defined(_WIN32) -+#ifdef HG_HAS_DIAG - hg_core_class_get_counters(&private_class->counters, diag_counters); - #else - HG_LOG_SUBSYS_ERROR(cls, "Counters not supported in current build, please " -diff --git a/src/na/CMakeLists.txt b/src/na/CMakeLists.txt -index 115e70bb..d99ed05d 100644 ---- a/src/na/CMakeLists.txt -+++ b/src/na/CMakeLists.txt -@@ -59,6 +59,11 @@ set(NA_BUILD_INCLUDE_DEPENDENCIES - #------------------------------------------------------------------------------ - # Internal dependencies - #------------------------------------------------------------------------------ -+# Diagnostics counters -+if(HG_HAS_DIAG) -+ set(NA_HAS_DIAG 1) -+endif() -+ - # Multi progress - if(NOT HG_ALLOW_MULTI_PROGRESS) - option(NA_ALLOW_MULTI_PROGRESS "Allow concurrent progress on single context." ON) -diff --git a/src/na/na.c b/src/na/na.c -index d54db626..61abf0a3 100644 ---- a/src/na/na.c -+++ b/src/na/na.c -@@ -1010,6 +1010,15 @@ NA_Set_log_level(const char *level) - hg_log_set_subsys_level(NA_SUBSYS_NAME_STRING, hg_log_name_to_level(level)); - } - -+/*---------------------------------------------------------------------------*/ -+void -+NA_Diag_dump_counters(void) -+{ -+#ifndef _WIN32 -+ hg_log_dump_counters(&HG_LOG_OUTLET(NA_SUBSYS_NAME)); -+#endif -+} -+ - /*---------------------------------------------------------------------------*/ - na_context_t * - NA_Context_create(na_class_t *na_class) -diff --git a/src/na/na.h b/src/na/na.h -index ff65ceb2..3d335fe7 100644 ---- a/src/na/na.h -+++ b/src/na/na.h -@@ -153,6 +153,12 @@ NA_Has_opt_feature( - NA_PUBLIC void - NA_Set_log_level(const char *level); - -+/** -+ * Dump diagnostic counters into the existing log stream. -+ */ -+NA_PUBLIC void -+NA_Diag_dump_counters(void); -+ - /** - * Return the name of the NA class. - * -diff --git a/src/na/na_config.h.in b/src/na/na_config.h.in -index 8468de8c..1419bee6 100644 ---- a/src/na/na_config.h.in -+++ b/src/na/na_config.h.in -@@ -85,6 +85,7 @@ - - /* Build Options */ - #cmakedefine NA_HAS_DEBUG -+#cmakedefine NA_HAS_DIAG - #cmakedefine NA_HAS_MULTI_PROGRESS - - /* HWLOC */ -diff --git a/src/na/na_ofi.c b/src/na/na_ofi.c -index 682efe65..d385d279 100644 ---- a/src/na/na_ofi.c -+++ b/src/na/na_ofi.c -@@ -262,6 +262,9 @@ static unsigned long const na_ofi_prov_flags[] = {NA_OFI_PROV_TYPES}; - /* Prov info array init count */ - #define NA_OFI_PROV_INFO_COUNT (32) - -+/* Max counter name length */ -+#define NA_OFI_MAX_COUNTER_NAME (64) -+ - /* Address / URI max len */ - #define NA_OFI_MAX_URI_LEN (128) - -@@ -787,7 +790,6 @@ struct na_ofi_domain { - hg_atomic_int64_t requested_key; /* Requested key if not FI_MR_PROV_KEY */ - int64_t max_key; /* Max key if not FI_MR_PROV_KEY */ - uint64_t max_tag; /* Max tag from CQ data size */ -- hg_atomic_int32_t mr_reg_count; /* Number of MR registered */ - bool no_wait; /* Wait disabled on domain */ - bool av_auth_key; /* Use FI_AV_AUTH_KEY */ - bool av_user_id; /* Use FI_AV_USER_ID */ -@@ -830,15 +832,26 @@ struct na_ofi_verify_info { - enum na_ofi_prov_type prov_type; /* Provider type */ - }; - --/* OFI class */ --struct na_ofi_class { -- struct na_ofi_addr_pool addr_pool; /* Addr pool */ -- struct fi_info *fi_info; /* OFI info */ -- struct na_ofi_fabric *fabric; /* Fabric pointer */ -- struct na_ofi_domain *domain; /* Domain pointer */ -- struct na_ofi_endpoint *endpoint; /* Endpoint pointer */ -- struct hg_mem_pool *send_pool; /* Msg send buf pool */ -- struct hg_mem_pool *recv_pool; /* Msg recv buf pool */ -+#ifdef NA_HAS_DIAG -+/* OFI counters */ -+struct na_ofi_counters { -+ char tx_count_string[NA_OFI_MAX_COUNTER_NAME]; /* TX count string */ -+ char rx_count_string[NA_OFI_MAX_COUNTER_NAME]; /* RX count string */ -+ char rma_count_string[NA_OFI_MAX_COUNTER_NAME]; /* RMA count string */ -+ char mr_count_string[NA_OFI_MAX_COUNTER_NAME]; /* MR count string */ -+ char addr_count_string[NA_OFI_MAX_COUNTER_NAME]; /* Addr count string */ -+ char cq_count_string[NA_OFI_MAX_COUNTER_NAME]; /* CQ count string */ -+ hg_atomic_int32_t *tx_count; /* Number of active sends */ -+ hg_atomic_int32_t *rx_count; /* Number of active receives */ -+ hg_atomic_int32_t *rma_count; /* Number of active RMAs */ -+ hg_atomic_int32_t *mr_count; /* Number of active MRs */ -+ hg_atomic_int32_t *addr_count; /* Number of addresses inserted */ -+ hg_atomic_int32_t *cq_count; /* Number of CQ events */ -+}; -+#endif -+ -+/* OFI ops */ -+struct na_ofi_ops { - na_return_t (*msg_send_unexpected)( - struct fid_ep *, const struct na_ofi_msg_info *, void *); - na_return_t (*msg_recv_unexpected)( -@@ -853,14 +866,29 @@ struct na_ofi_class { - const char *msg_recv_unexpected_string; /* Error log string */ - const char *msg_send_expected_string; /* Error log string */ - const char *msg_recv_expected_string; /* Error log string */ -- unsigned long opt_features; /* Optional feature flags */ -- hg_atomic_int32_t n_contexts; /* Number of context */ -- unsigned int op_retry_timeout; /* Retry timeout */ -- unsigned int op_retry_period; /* Time elapsed until next retry */ -- uint8_t context_max; /* Max number of contexts */ -- bool no_wait; /* Ignore wait object */ -- bool use_sep; /* Use scalable endpoints */ -- bool finalizing; /* Class being destroyed */ -+}; -+ -+/* OFI class */ -+struct na_ofi_class { -+ struct na_ofi_addr_pool addr_pool; /* Addr pool */ -+ struct fi_info *fi_info; /* OFI info */ -+ struct na_ofi_fabric *fabric; /* Fabric pointer */ -+ struct na_ofi_domain *domain; /* Domain pointer */ -+ struct na_ofi_endpoint *endpoint; /* Endpoint pointer */ -+ struct hg_mem_pool *send_pool; /* Msg send buf pool */ -+ struct hg_mem_pool *recv_pool; /* Msg recv buf pool */ -+ struct na_ofi_ops ops; /* OFI operations */ -+#ifdef NA_HAS_DIAG -+ struct na_ofi_counters counters; /* OFI counters */ -+#endif -+ unsigned long opt_features; /* Optional feature flags */ -+ hg_atomic_int32_t n_contexts; /* Number of context */ -+ unsigned int op_retry_timeout; /* Retry timeout */ -+ unsigned int op_retry_period; /* Time elapsed until next retry */ -+ uint8_t context_max; /* Max number of contexts */ -+ bool no_wait; /* Ignore wait object */ -+ bool use_sep; /* Use scalable endpoints */ -+ bool finalizing; /* Class being destroyed */ - }; - - /********************/ -@@ -1145,6 +1173,20 @@ na_ofi_class_alloc(void); - static na_return_t - na_ofi_class_free(struct na_ofi_class *na_ofi_class); - -+#ifdef NA_HAS_DIAG -+/** -+ * Init counters. -+ */ -+static void -+na_ofi_counters_init(struct na_ofi_counters *counters, int class_id); -+ -+/** -+ * Finalize counters. -+ */ -+static void -+na_ofi_counters_finalize(struct na_ofi_counters *counters); -+#endif -+ - /** - * Configure class parameters from environment variables. - */ -@@ -1762,6 +1804,14 @@ static void - na_ofi_op_retry_abort_addr( - struct na_ofi_context *na_ofi_context, fi_addr_t fi_addr, na_return_t ret); - -+/** -+ * Process counters. -+ */ -+#ifdef NA_HAS_DIAG -+static void -+na_ofi_cq_process_counters(struct na_ofi_op_id *na_ofi_op_id); -+#endif -+ - /** - * Complete operation ID. - */ -@@ -3307,6 +3357,12 @@ na_ofi_addr_map_insert(struct na_ofi_class *na_ofi_class, - na_ofi_errno_to_na(-rc), - "fi_av_remove(%" PRIu64 ") failed, rc: %d (%s)", - na_ofi_addr->fi_addr, rc, fi_strerror(-rc)); -+ -+#ifdef NA_HAS_DIAG -+ /* Counters */ -+ hg_atomic_decr32(na_ofi_class->counters.addr_count); -+#endif -+ - addr_map_exist = true; - } - } else { -@@ -3353,6 +3409,11 @@ na_ofi_addr_map_insert(struct na_ofi_class *na_ofi_class, - addr_str, &addr_str_len), - rc); - -+#ifdef NA_HAS_DIAG -+ /* Counters */ -+ hg_atomic_incr32(na_ofi_class->counters.addr_count); -+#endif -+ - #if FI_VERSION_GE(FI_COMPILE_VERSION, FI_VERSION(1, 20)) - if (na_ofi_class->domain->av_auth_key) { - size_t addrlen = sizeof(na_ofi_addr->addr_key.addr); -@@ -3458,6 +3519,11 @@ na_ofi_addr_map_remove( - "fi_av_remove(%" PRIu64 ") failed, rc: %d (%s)", na_ofi_addr->fi_addr, - rc, fi_strerror(-rc)); - -+#ifdef NA_HAS_DIAG -+ /* Counters */ -+ hg_atomic_decr32(na_ofi_addr->class->counters.addr_count); -+#endif -+ - NA_LOG_SUBSYS_DEBUG( - addr, "Removed addr for FI addr %" PRIu64, na_ofi_addr->fi_addr); - -@@ -4229,6 +4295,9 @@ static struct na_ofi_class * - na_ofi_class_alloc(void) - { - struct na_ofi_class *na_ofi_class = NULL; -+#ifdef NA_HAS_DIAG -+ static int class_id = 0; -+#endif - int rc; - - /* Create private data */ -@@ -4237,6 +4306,10 @@ na_ofi_class_alloc(void) - "Could not allocate NA private data class"); - hg_atomic_init32(&na_ofi_class->n_contexts, 0); - -+#ifdef NA_HAS_DIAG -+ na_ofi_counters_init(&na_ofi_class->counters, class_id++); -+#endif -+ - /* Initialize addr pool */ - rc = hg_thread_spin_init(&na_ofi_class->addr_pool.lock); - NA_CHECK_SUBSYS_ERROR_NORET( -@@ -4301,6 +4374,11 @@ na_ofi_class_free(struct na_ofi_class *na_ofi_class) - na_ofi_class->fabric = NULL; - } - -+#ifdef NA_HAS_DIAG -+ /* Remove counters */ -+ na_ofi_counters_finalize(&na_ofi_class->counters); -+#endif -+ - /* Free info */ - if (na_ofi_class->fi_info) - na_ofi_freeinfo(na_ofi_class->fi_info); -@@ -4313,6 +4391,50 @@ out: - return ret; - } - -+/*---------------------------------------------------------------------------*/ -+#ifdef NA_HAS_DIAG -+static void -+na_ofi_counters_init(struct na_ofi_counters *counters, int class_id) -+{ -+ snprintf(counters->tx_count_string, sizeof(counters->tx_count_string), -+ "[%d] na_ofi_tx_count ", class_id); -+ snprintf(counters->rx_count_string, sizeof(counters->rx_count_string), -+ "[%d] na_ofi_rx_count ", class_id); -+ snprintf(counters->rma_count_string, sizeof(counters->rma_count_string), -+ "[%d] na_ofi_rma_count ", class_id); -+ snprintf(counters->mr_count_string, sizeof(counters->mr_count_string), -+ "[%d] na_ofi_mr_count ", class_id); -+ snprintf(counters->addr_count_string, sizeof(counters->addr_count_string), -+ "[%d] na_ofi_addr_count", class_id); -+ snprintf(counters->cq_count_string, sizeof(counters->cq_count_string), -+ "[%d] na_ofi_cq_count ", class_id); -+ HG_LOG_ADD_COUNTER32(na, &counters->tx_count, counters->tx_count_string, -+ "Number of active sends"); -+ HG_LOG_ADD_COUNTER32(na, &counters->rx_count, counters->rx_count_string, -+ "Number of active recvs"); -+ HG_LOG_ADD_COUNTER32(na, &counters->rma_count, counters->rma_count_string, -+ "Number of active RMAs"); -+ HG_LOG_ADD_COUNTER32(na, &counters->mr_count, counters->mr_count_string, -+ "Number of active MRs"); -+ HG_LOG_ADD_COUNTER32(na, &counters->addr_count, counters->addr_count_string, -+ "Number of addresses inserted"); -+ HG_LOG_ADD_COUNTER32(na, &counters->cq_count, counters->cq_count_string, -+ "Number of events still in CQ"); -+} -+ -+/*---------------------------------------------------------------------------*/ -+static void -+na_ofi_counters_finalize(struct na_ofi_counters *counters) -+{ -+ HG_LOG_DEL_COUNTER32(na, counters->tx_count); -+ HG_LOG_DEL_COUNTER32(na, counters->rx_count); -+ HG_LOG_DEL_COUNTER32(na, counters->rma_count); -+ HG_LOG_DEL_COUNTER32(na, counters->mr_count); -+ HG_LOG_DEL_COUNTER32(na, counters->addr_count); -+ HG_LOG_DEL_COUNTER32(na, counters->cq_count); -+} -+#endif -+ - /*---------------------------------------------------------------------------*/ - static na_return_t - na_ofi_class_env_config(struct na_ofi_class *na_ofi_class) -@@ -4323,26 +4445,26 @@ na_ofi_class_env_config(struct na_ofi_class *na_ofi_class) - /* Set unexpected msg callbacks */ - env = getenv("NA_OFI_UNEXPECTED_TAG_MSG"); - if (env == NULL || env[0] == '0' || tolower(env[0]) == 'n') { -- na_ofi_class->msg_send_unexpected = na_ofi_msg_send; -- na_ofi_class->msg_send_unexpected_string = "fi_senddata"; -- na_ofi_class->msg_recv_unexpected = na_ofi_msg_recv; -- na_ofi_class->msg_recv_unexpected_string = "fi_recv"; -+ na_ofi_class->ops.msg_send_unexpected = na_ofi_msg_send; -+ na_ofi_class->ops.msg_send_unexpected_string = "fi_senddata"; -+ na_ofi_class->ops.msg_recv_unexpected = na_ofi_msg_recv; -+ na_ofi_class->ops.msg_recv_unexpected_string = "fi_recv"; - } else { - NA_LOG_SUBSYS_DEBUG(cls, - "NA_OFI_UNEXPECTED_TAG_MSG set to %s, forcing unexpected messages " - "to use tagged recvs", - env); -- na_ofi_class->msg_send_unexpected = na_ofi_tag_send; -- na_ofi_class->msg_send_unexpected_string = "fi_tsend"; -- na_ofi_class->msg_recv_unexpected = na_ofi_tag_recv; -- na_ofi_class->msg_recv_unexpected_string = "fi_trecv"; -+ na_ofi_class->ops.msg_send_unexpected = na_ofi_tag_send; -+ na_ofi_class->ops.msg_send_unexpected_string = "fi_tsend"; -+ na_ofi_class->ops.msg_recv_unexpected = na_ofi_tag_recv; -+ na_ofi_class->ops.msg_recv_unexpected_string = "fi_trecv"; - } - - /* Set expected msg callbacks */ -- na_ofi_class->msg_send_expected = na_ofi_tag_send; -- na_ofi_class->msg_send_expected_string = "fi_tsend"; -- na_ofi_class->msg_recv_expected = na_ofi_tag_recv; -- na_ofi_class->msg_recv_expected_string = "fi_trecv"; -+ na_ofi_class->ops.msg_send_expected = na_ofi_tag_send; -+ na_ofi_class->ops.msg_send_expected_string = "fi_tsend"; -+ na_ofi_class->ops.msg_recv_expected = na_ofi_tag_recv; -+ na_ofi_class->ops.msg_recv_expected_string = "fi_trecv"; - - /* Default retry timeouts in ms */ - if ((env = getenv("NA_OFI_OP_RETRY_TIMEOUT")) != NULL) { -@@ -5073,7 +5195,6 @@ na_ofi_domain_open(const struct na_ofi_fabric *na_ofi_fabric, - hg_atomic_init64(&na_ofi_domain->requested_key, 0); - /* No need to take a refcount on fabric */ - na_ofi_domain->fabric = na_ofi_fabric; -- hg_atomic_init32(&na_ofi_domain->mr_reg_count, 0); - - /* Dup name */ - na_ofi_domain->name = strdup(domain_attr->name); -@@ -6056,6 +6177,11 @@ na_ofi_mem_buf_register(const void *buf, size_t len, unsigned long flags, - - /* Register memory if FI_MR_LOCAL is set and provider uses it */ - if (na_ofi_class->fi_info->domain_attr->mr_mode & FI_MR_LOCAL) { -+#ifdef NA_HAS_DIAG -+ int32_t mr_cnt = hg_atomic_get32(na_ofi_class->counters.mr_count); -+#else -+ int32_t mr_cnt = -1; -+#endif - struct fid_mr *mr_hdl = NULL; - uint64_t access = 0; - int rc; -@@ -6072,10 +6198,11 @@ na_ofi_mem_buf_register(const void *buf, size_t len, unsigned long flags, - NA_CHECK_SUBSYS_ERROR(mem, rc != 0, out, ret, HG_UTIL_FAIL, - "fi_mr_reg(buf=%p, len=%zu, flags=%lu) failed, rc: %d (%s), " - "mr_reg_count: %d", -- buf, len, flags, rc, fi_strerror(-rc), -- hg_atomic_get32(&na_ofi_class->domain->mr_reg_count)); -+ buf, len, flags, rc, fi_strerror(-rc), mr_cnt); - -- hg_atomic_incr32(&na_ofi_class->domain->mr_reg_count); -+#ifdef NA_HAS_DIAG -+ hg_atomic_incr32(na_ofi_class->counters.mr_count); -+#endif - *handle_p = (void *) mr_hdl; - } else - *handle_p = NULL; -@@ -6093,11 +6220,17 @@ na_ofi_mem_buf_deregister(void *handle, void *arg) - /* Release MR handle is there was any */ - if (handle) { - struct fid_mr *mr_hdl = (struct fid_mr *) handle; -+#ifdef NA_HAS_DIAG - struct na_ofi_class *na_ofi_class = (struct na_ofi_class *) arg; -+#else -+ (void) arg; -+#endif - int rc = fi_close(&mr_hdl->fid); - NA_CHECK_SUBSYS_ERROR(mem, rc != 0, out, ret, HG_UTIL_FAIL, - "fi_close() mr_hdl failed, rc: %d (%s)", rc, fi_strerror(-rc)); -- hg_atomic_decr32(&na_ofi_class->domain->mr_reg_count); -+#ifdef NA_HAS_DIAG -+ hg_atomic_decr32(na_ofi_class->counters.mr_count); -+#endif - } - - out: -@@ -6159,6 +6292,11 @@ na_ofi_msg_send_common(struct na_ofi_class *na_ofi_class, - if ((int) na_ofi_class->fi_info->addr_format == FI_ADDR_OPX) - na_ofi_op_id->fi_ctx[0].internal[0] = &na_ofi_addr->addr_key.addr.opx; - -+#ifdef NA_HAS_DIAG -+ /* Counters */ -+ hg_atomic_incr32(na_ofi_class->counters.tx_count); -+#endif -+ - ret = msg_op( - na_ofi_context->fi_tx, &na_ofi_op_id->info.msg, &na_ofi_op_id->fi_ctx); - if (ret != NA_SUCCESS) { -@@ -6166,8 +6304,12 @@ na_ofi_msg_send_common(struct na_ofi_class *na_ofi_class, - na_ofi_op_id->retry_op.msg = msg_op; - na_ofi_op_retry( - na_ofi_context, na_ofi_class->op_retry_timeout, na_ofi_op_id); -- } else -+ } else { -+#ifdef NA_HAS_DIAG -+ hg_atomic_decr32(na_ofi_class->counters.tx_count); -+#endif - NA_GOTO_SUBSYS_ERROR_NORET(msg, release, "Could not post msg send"); -+ } - } - - return NA_SUCCESS; -@@ -6224,6 +6366,11 @@ na_ofi_msg_recv_common(struct na_ofi_class *na_ofi_class, - .tag = tag, - .tag_mask = tag_mask}; - -+#ifdef NA_HAS_DIAG -+ /* Counters */ -+ hg_atomic_incr32(na_ofi_class->counters.rx_count); -+#endif -+ - ret = msg_op( - na_ofi_context->fi_rx, &na_ofi_op_id->info.msg, &na_ofi_op_id->fi_ctx); - if (ret != NA_SUCCESS) { -@@ -6231,8 +6378,12 @@ na_ofi_msg_recv_common(struct na_ofi_class *na_ofi_class, - na_ofi_op_id->retry_op.msg = msg_op; - na_ofi_op_retry( - na_ofi_context, na_ofi_class->op_retry_timeout, na_ofi_op_id); -- } else -+ } else { -+#ifdef NA_HAS_DIAG -+ hg_atomic_decr32(na_ofi_class->counters.rx_count); -+#endif - NA_GOTO_SUBSYS_ERROR_NORET(msg, release, "Could not post msg recv"); -+ } - } - - return NA_SUCCESS; -@@ -6620,6 +6771,11 @@ na_ofi_rma_common(struct na_ofi_class *na_ofi_class, na_context_t *context, - NA_OFI_SEP_RX_CTX_BITS) - : na_ofi_addr->fi_addr; - -+#ifdef NA_HAS_DIAG -+ /* Counters */ -+ hg_atomic_incr32(na_ofi_class->counters.rma_count); -+#endif -+ - /* Post the OFI RMA operation */ - ret = - na_ofi_rma_post(na_ofi_context->fi_tx, rma_info, &na_ofi_op_id->fi_ctx); -@@ -6628,8 +6784,12 @@ na_ofi_rma_common(struct na_ofi_class *na_ofi_class, na_context_t *context, - na_ofi_op_id->retry_op.rma = na_ofi_rma_post; - na_ofi_op_retry( - na_ofi_context, na_ofi_class->op_retry_timeout, na_ofi_op_id); -- } else -+ } else { -+#ifdef NA_HAS_DIAG -+ hg_atomic_decr32(na_ofi_class->counters.rma_count); -+#endif - NA_GOTO_SUBSYS_ERROR_NORET(rma, release, "Could not post RMA op"); -+ } - } - - return NA_SUCCESS; -@@ -6991,6 +7151,10 @@ na_ofi_cq_process_canceled(const struct na_ofi_class *na_ofi_class, - cq_err->err, fi_strerror(cq_err->err), (void *) na_ofi_op_id, - na_cb_type_to_string(na_ofi_op_id->type)); - -+#ifdef NA_HAS_DIAG -+ na_ofi_cq_process_counters(na_ofi_op_id); -+#endif -+ - /* When tearing down connections, it is possible that operations will be - canceled by libfabric itself. - NA_CHECK_SUBSYS_WARNING(op, -@@ -7014,6 +7178,10 @@ na_ofi_cq_process_canceled(const struct na_ofi_class *na_ofi_class, - &na_ofi_op_id->completion_data->callback_info.info - .multi_recv_unexpected, - complete); -+#ifdef NA_HAS_DIAG -+ if (complete) -+ hg_atomic_decr32(na_ofi_class->counters.rx_count); -+#endif - } else - complete = true; - -@@ -7151,6 +7319,10 @@ na_ofi_cq_process_error( - NA_OFI_OP_CANCELED) - return NA_SUCCESS; /* already handled */ - -+#ifdef NA_HAS_DIAG -+ na_ofi_cq_process_counters(na_ofi_op_id); -+#endif -+ - /* Abort other retries if peer is unreachable */ - if (na_ret == NA_HOSTUNREACH && na_ofi_op_id->addr) - na_ofi_op_retry_abort_addr(na_ofi_op_id->na_ofi_context, -@@ -7172,6 +7344,10 @@ na_ofi_cq_process_error( - &na_ofi_op_id->completion_data->callback_info.info - .multi_recv_unexpected, - complete); -+#ifdef NA_HAS_DIAG -+ if (complete) -+ hg_atomic_decr32(na_ofi_class->counters.rx_count); -+#endif - } else - complete = true; - -@@ -7346,6 +7522,10 @@ na_ofi_cq_process_event(struct na_ofi_class *na_ofi_class, - cq_event->op_context, cq_event->flags, cq_event->len, cq_event->buf, - cq_event->data, cq_event->tag); - -+#ifdef NA_HAS_DIAG -+ na_ofi_cq_process_counters(na_ofi_op_id); -+#endif -+ - switch (na_ofi_op_id->type) { - case NA_CB_RECV_UNEXPECTED: - /* Default to cq_event->tag for backward compatibility */ -@@ -7361,6 +7541,10 @@ na_ofi_cq_process_event(struct na_ofi_class *na_ofi_class, - break; - case NA_CB_MULTI_RECV_UNEXPECTED: - complete = cq_event->flags & FI_MULTI_RECV; -+#ifdef HG_HAS_DIAG -+ if (complete) -+ hg_atomic_decr32(na_ofi_class->counters.rx_count); -+#endif - - ret = na_ofi_cq_process_multi_recv_unexpected(na_ofi_class, - &na_ofi_op_id->info.msg, -@@ -7628,6 +7812,10 @@ na_ofi_cq_process_retries( - NA_LOG_SUBSYS_ERROR(op, "retry operation of %p (%s) failed", - (void *) na_ofi_op_id, na_cb_type_to_string(cb_type)); - -+#ifdef NA_HAS_DIAG -+ na_ofi_cq_process_counters(na_ofi_op_id); -+#endif -+ - /* Force internal completion in error mode */ - hg_atomic_or32(&na_ofi_op_id->status, NA_OFI_OP_ERRORED); - na_ofi_op_id->complete(na_ofi_op_id, true, ret); -@@ -7690,6 +7878,32 @@ na_ofi_op_retry_abort_addr( - hg_thread_spin_unlock(&op_queue->lock); - } - -+/*---------------------------------------------------------------------------*/ -+#ifdef NA_HAS_DIAG -+static void -+na_ofi_cq_process_counters(struct na_ofi_op_id *na_ofi_op_id) -+{ -+ switch (na_ofi_op_id->type) { -+ case NA_CB_RECV_UNEXPECTED: -+ case NA_CB_RECV_EXPECTED: -+ hg_atomic_decr32(na_ofi_op_id->na_ofi_class->counters.rx_count); -+ break; -+ case NA_CB_SEND_UNEXPECTED: -+ case NA_CB_SEND_EXPECTED: -+ hg_atomic_decr32(na_ofi_op_id->na_ofi_class->counters.tx_count); -+ break; -+ case NA_CB_PUT: -+ case NA_CB_GET: -+ hg_atomic_decr32(na_ofi_op_id->na_ofi_class->counters.rma_count); -+ break; -+ case NA_CB_MULTI_RECV_UNEXPECTED: -+ /* TODO currently treated outside of switch */ -+ default: -+ break; -+ } -+} -+#endif -+ - /*---------------------------------------------------------------------------*/ - static NA_INLINE void - na_ofi_op_complete_single(struct na_ofi_op_id *na_ofi_op_id, -@@ -7711,6 +7925,9 @@ na_ofi_op_complete_single(struct na_ofi_op_id *na_ofi_op_id, - completion_data->plugin_callback = na_ofi_op_release_single; - - NA_LOG_SUBSYS_DEBUG(op, "Adding completion data to queue"); -+#ifdef NA_HAS_DIAG -+ hg_atomic_incr32(na_ofi_op_id->na_ofi_class->counters.cq_count); -+#endif - - /* Add OP to NA completion queue */ - na_cb_completion_add( -@@ -7728,6 +7945,9 @@ na_ofi_op_release_single(void *arg) - (!(hg_atomic_get32(&na_ofi_op_id->status) & NA_OFI_OP_COMPLETED)), - "Releasing resources from an uncompleted operation"); - -+#ifdef NA_HAS_DIAG -+ hg_atomic_decr32(na_ofi_op_id->na_ofi_class->counters.cq_count); -+#endif - if (na_ofi_op_id->addr) { - na_ofi_addr_ref_decr(na_ofi_op_id->addr); - na_ofi_op_id->addr = NULL; -@@ -7775,6 +7995,10 @@ na_ofi_op_complete_multi( - op, na_ofi_op_id->completion_data == NULL, error, "Queue is full"); - - NA_LOG_SUBSYS_DEBUG(op, "Adding completion data to queue"); -+#ifdef NA_HAS_DIAG -+ hg_atomic_incr32(na_ofi_op_id->na_ofi_class->counters.cq_count); -+#endif -+ - /* Add OP to NA completion queue */ - na_cb_completion_add( - na_ofi_op_id->na_ofi_context->context, completion_data); -@@ -7789,6 +8013,9 @@ na_ofi_op_release_multi(void *arg) - { - struct na_ofi_op_id *na_ofi_op_id = (struct na_ofi_op_id *) arg; - -+#ifdef NA_HAS_DIAG -+ hg_atomic_decr32(na_ofi_op_id->na_ofi_class->counters.cq_count); -+#endif - na_ofi_completion_multi_pop(&na_ofi_op_id->completion_data_storage.multi); - } - -@@ -8230,15 +8457,15 @@ na_ofi_initialize( - - /* Set/check optional features */ - if ((na_ofi_prov_extra_caps[prov_type] & FI_MULTI_RECV) && -- (na_ofi_class->msg_recv_unexpected == na_ofi_msg_recv)) { -+ (na_ofi_class->ops.msg_recv_unexpected == na_ofi_msg_recv)) { - NA_CHECK_SUBSYS_ERROR(cls, - !(na_ofi_class->fi_info->caps & FI_MULTI_RECV), error, ret, - NA_PROTONOSUPPORT, "FI_MULTI_RECV is not supported by provider"); - na_ofi_class->opt_features |= NA_OPT_MULTI_RECV; - } -- na_ofi_class->cq_poll = (na_ofi_class->fi_info->caps & FI_SOURCE_ERR) -- ? na_ofi_cq_poll_fi_source -- : na_ofi_cq_poll_no_source; -+ na_ofi_class->ops.cq_poll = (na_ofi_class->fi_info->caps & FI_SOURCE_ERR) -+ ? na_ofi_cq_poll_fi_source -+ : na_ofi_cq_poll_no_source; - - /* Open fabric */ - ret = na_ofi_fabric_open( -@@ -8959,8 +9186,8 @@ na_ofi_msg_send_unexpected(na_class_t *na_class, na_context_t *context, - { - return na_ofi_msg_send_common(NA_OFI_CLASS(na_class), - NA_OFI_CONTEXT(context), NA_CB_SEND_UNEXPECTED, callback, arg, -- NA_OFI_CLASS(na_class)->msg_send_unexpected, -- NA_OFI_CLASS(na_class)->msg_send_unexpected_string, buf, buf_size, -+ NA_OFI_CLASS(na_class)->ops.msg_send_unexpected, -+ NA_OFI_CLASS(na_class)->ops.msg_send_unexpected_string, buf, buf_size, - NA_OFI_CLASS(na_class)->endpoint->unexpected_msg_size_max, - (struct na_ofi_msg_buf_handle *) plugin_data, - (struct na_ofi_addr *) dest_addr, dest_id, -@@ -8975,8 +9202,8 @@ na_ofi_msg_recv_unexpected(na_class_t *na_class, na_context_t *context, - { - return na_ofi_msg_recv_common(NA_OFI_CLASS(na_class), - NA_OFI_CONTEXT(context), NA_CB_RECV_UNEXPECTED, callback, arg, -- NA_OFI_CLASS(na_class)->msg_recv_unexpected, -- NA_OFI_CLASS(na_class)->msg_recv_unexpected_string, buf, buf_size, -+ NA_OFI_CLASS(na_class)->ops.msg_recv_unexpected, -+ NA_OFI_CLASS(na_class)->ops.msg_recv_unexpected_string, buf, buf_size, - NA_OFI_CLASS(na_class)->endpoint->unexpected_msg_size_max, - (struct na_ofi_msg_buf_handle *) plugin_data, NULL, 0, - NA_OFI_UNEXPECTED_TAG, NA_OFI_TAG_MASK, (struct na_ofi_op_id *) op_id); -@@ -9027,9 +9254,17 @@ na_ofi_msg_multi_recv_unexpected(na_class_t *na_class, na_context_t *context, - .tag = 0 /* unused */, - .tag_mask = 0 /* unused */}; - -+#ifdef NA_HAS_DIAG -+ /* Counters */ -+ hg_atomic_incr32(na_ofi_class->counters.rx_count); -+#endif -+ - ret = na_ofi_msg_multi_recv( - na_ofi_context->fi_rx, &na_ofi_op_id->info.msg, &na_ofi_op_id->fi_ctx); - if (ret != NA_SUCCESS) { -+#ifdef NA_HAS_DIAG -+ hg_atomic_decr32(na_ofi_class->counters.rx_count); -+#endif - if (ret == NA_AGAIN) { - na_ofi_op_id->retry_op.msg = na_ofi_msg_multi_recv; - na_ofi_op_retry( -@@ -9061,8 +9296,8 @@ na_ofi_msg_send_expected(na_class_t *na_class, na_context_t *context, - { - return na_ofi_msg_send_common(NA_OFI_CLASS(na_class), - NA_OFI_CONTEXT(context), NA_CB_SEND_EXPECTED, callback, arg, -- NA_OFI_CLASS(na_class)->msg_send_expected, -- NA_OFI_CLASS(na_class)->msg_send_expected_string, buf, buf_size, -+ NA_OFI_CLASS(na_class)->ops.msg_send_expected, -+ NA_OFI_CLASS(na_class)->ops.msg_send_expected_string, buf, buf_size, - NA_OFI_CLASS(na_class)->endpoint->expected_msg_size_max, - (struct na_ofi_msg_buf_handle *) plugin_data, - (struct na_ofi_addr *) dest_addr, dest_id, (uint64_t) tag, -@@ -9077,8 +9312,8 @@ na_ofi_msg_recv_expected(na_class_t *na_class, na_context_t *context, - { - return na_ofi_msg_recv_common(NA_OFI_CLASS(na_class), - NA_OFI_CONTEXT(context), NA_CB_RECV_EXPECTED, callback, arg, -- NA_OFI_CLASS(na_class)->msg_recv_expected, -- NA_OFI_CLASS(na_class)->msg_recv_expected_string, buf, buf_size, -+ NA_OFI_CLASS(na_class)->ops.msg_recv_expected, -+ NA_OFI_CLASS(na_class)->ops.msg_recv_expected_string, buf, buf_size, - NA_OFI_CLASS(na_class)->endpoint->expected_msg_size_max, - (struct na_ofi_msg_buf_handle *) plugin_data, - (struct na_ofi_addr *) source_addr, source_id, (uint64_t) tag, 0, -@@ -9214,7 +9449,11 @@ na_ofi_mem_register(na_class_t *na_class, na_mem_handle_t *mem_handle, - (struct na_ofi_mem_handle *) mem_handle; - struct na_ofi_domain *domain = NA_OFI_CLASS(na_class)->domain; - const struct fi_info *fi_info = NA_OFI_CLASS(na_class)->fi_info; -- int32_t mr_cnt = hg_atomic_get32(&domain->mr_reg_count); -+#ifdef NA_HAS_DIAG -+ int32_t mr_cnt = hg_atomic_get32(NA_OFI_CLASS(na_class)->counters.mr_count); -+#else -+ int32_t mr_cnt = -1; -+#endif - struct fi_mr_attr fi_mr_attr = { - .mr_iov = NA_OFI_IOV( - na_ofi_mem_handle->desc.iov, na_ofi_mem_handle->desc.info.iovcnt), -@@ -9283,7 +9522,9 @@ na_ofi_mem_register(na_class_t *na_class, na_mem_handle_t *mem_handle, - fi_mr_attr.mr_iov[0].iov_base, fi_mr_attr.mr_iov[0].iov_len, - fi_mr_attr.iov_count, fi_mr_attr.access, fi_mr_attr.iface, - fi_mr_attr.requested_key, rc, fi_strerror(-rc), mr_cnt); -- mr_cnt = hg_atomic_incr32(&domain->mr_reg_count); -+#ifdef NA_HAS_DIAG -+ mr_cnt = hg_atomic_incr32(NA_OFI_CLASS(na_class)->counters.mr_count); -+#endif - - /* Attach MR to endpoint when provider requests it */ - if (fi_info->domain_attr->mr_mode & FI_MR_ENDPOINT) { -@@ -9327,7 +9568,9 @@ na_ofi_mem_register(na_class_t *na_class, na_mem_handle_t *mem_handle, - error: - if (na_ofi_mem_handle->fi_mr) { - (void) fi_close(&na_ofi_mem_handle->fi_mr->fid); -- hg_atomic_decr32(&domain->mr_reg_count); -+#ifdef NA_HAS_DIAG -+ hg_atomic_decr32(NA_OFI_CLASS(na_class)->counters.mr_count); -+#endif - } - return ret; - } -@@ -9336,7 +9579,6 @@ error: - static na_return_t - na_ofi_mem_deregister(na_class_t *na_class, na_mem_handle_t *mem_handle) - { -- struct na_ofi_domain *domain = NA_OFI_CLASS(na_class)->domain; - struct na_ofi_mem_handle *na_ofi_mem_handle = - (struct na_ofi_mem_handle *) mem_handle; - na_return_t ret; -@@ -9344,14 +9586,18 @@ na_ofi_mem_deregister(na_class_t *na_class, na_mem_handle_t *mem_handle) - - /* close MR handle */ - if (na_ofi_mem_handle->fi_mr != NULL) { -- int32_t NA_DEBUG_LOG_USED mr_cnt; -+ int32_t NA_DEBUG_LOG_USED mr_cnt = -1; - const struct iovec NA_DEBUG_LOG_USED *mr_iov = NA_OFI_IOV( - na_ofi_mem_handle->desc.iov, na_ofi_mem_handle->desc.info.iovcnt); - - rc = fi_close(&na_ofi_mem_handle->fi_mr->fid); - NA_CHECK_SUBSYS_ERROR(mem, rc != 0, error, ret, na_ofi_errno_to_na(-rc), - "fi_close() mr_hdl failed, rc: %d (%s)", rc, fi_strerror(-rc)); -- mr_cnt = hg_atomic_decr32(&domain->mr_reg_count); -+#ifdef NA_HAS_DIAG -+ mr_cnt = hg_atomic_decr32(NA_OFI_CLASS(na_class)->counters.mr_count); -+#else -+ (void) na_class; -+#endif - - NA_LOG_SUBSYS_DEBUG(mem, - "Deregistered memory region: mr_iov[0].iov_base=%p, " -@@ -9575,7 +9821,7 @@ na_ofi_poll(na_class_t *na_class, na_context_t *context, unsigned int *count_p) - return NA_SUCCESS; - - /* Read from CQ and process events */ -- ret = na_ofi_class->cq_poll(na_ofi_class, na_ofi_context, &count); -+ ret = na_ofi_class->ops.cq_poll(na_ofi_class, na_ofi_context, &count); - NA_CHECK_SUBSYS_NA_ERROR(poll, error, ret, "Could not poll context CQ"); - - /* Attempt to process retries */ -diff --git a/src/util/mercury_dlog.c b/src/util/mercury_dlog.c -index 042a0157..fbdc114f 100644 ---- a/src/util/mercury_dlog.c -+++ b/src/util/mercury_dlog.c -@@ -138,6 +138,40 @@ hg_dlog_mkcount64(struct hg_dlog *d, hg_atomic_int64_t **cptr, const char *name, - hg_thread_mutex_unlock(&d->dlock); - } - -+/*---------------------------------------------------------------------------*/ -+void -+hg_dlog_rmcount32(struct hg_dlog *d, hg_atomic_int32_t *cptr) -+{ -+ struct hg_dlog_dcount32 *dcnt; -+ -+ hg_thread_mutex_lock(&d->dlock); -+ TAILQ_FOREACH (dcnt, &d->cnts32, l) { -+ if (&dcnt->c == cptr) { -+ TAILQ_REMOVE(&d->cnts32, dcnt, l); -+ free(dcnt); -+ break; -+ } -+ } -+ hg_thread_mutex_unlock(&d->dlock); -+} -+ -+/*---------------------------------------------------------------------------*/ -+void -+hg_dlog_rmcount64(struct hg_dlog *d, hg_atomic_int64_t *cptr) -+{ -+ struct hg_dlog_dcount64 *dcnt; -+ -+ hg_thread_mutex_lock(&d->dlock); -+ TAILQ_FOREACH (dcnt, &d->cnts64, l) { -+ if (&dcnt->c == cptr) { -+ TAILQ_REMOVE(&d->cnts64, dcnt, l); -+ free(dcnt); -+ break; -+ } -+ } -+ hg_thread_mutex_unlock(&d->dlock); -+} -+ - /*---------------------------------------------------------------------------*/ - unsigned int - hg_dlog_addlog(struct hg_dlog *d, const char *file, unsigned int line, -@@ -210,8 +244,9 @@ hg_dlog_dump(struct hg_dlog *d, int (*log_func)(FILE *, const char *, ...), - "### (%s) debug log summary\n" - "### ----------------------\n", - (d->dlog_magic + strlen(HG_DLOG_STDMAGIC))); -- if (!TAILQ_EMPTY(&d->cnts32) && !TAILQ_EMPTY(&d->cnts64)) { -+ if (!TAILQ_EMPTY(&d->cnts32) || !TAILQ_EMPTY(&d->cnts64)) { - log_func(stream, "# Counters\n"); -+ - TAILQ_FOREACH (dc32, &d->cnts32, l) { - log_func(stream, "# %s: %" PRId32 " [%s]\n", dc32->name, - hg_atomic_get32(&dc32->c), dc32->descr); -diff --git a/src/util/mercury_dlog.h b/src/util/mercury_dlog.h -index 88944b05..43c30a4f 100644 ---- a/src/util/mercury_dlog.h -+++ b/src/util/mercury_dlog.h -@@ -155,6 +155,15 @@ HG_UTIL_PUBLIC void - hg_dlog_mkcount32(struct hg_dlog *d, hg_atomic_int32_t **cptr, const char *name, - const char *descr); - -+/** -+ * remove a 32-bit counter from a dlog. -+ * -+ * \param d [IN] dlog to remove the counter from -+ * \param cptr [IN] pointer to counter to remove -+ */ -+HG_UTIL_PUBLIC void -+hg_dlog_rmcount32(struct hg_dlog *d, hg_atomic_int32_t *cptr); -+ - /** - * make a named atomic64 counter in a dlog and return a pointer to - * it. we use the dlock to ensure a counter under a given name only -@@ -178,6 +187,15 @@ HG_UTIL_PUBLIC void - hg_dlog_mkcount64(struct hg_dlog *d, hg_atomic_int64_t **cptr, const char *name, - const char *descr); - -+/** -+ * remove a 64-bit counter from a dlog. -+ * -+ * \param d [IN] dlog to remove the counter from -+ * \param cptr [IN] pointer to counter to remove -+ */ -+HG_UTIL_PUBLIC void -+hg_dlog_rmcount64(struct hg_dlog *d, hg_atomic_int64_t *cptr); -+ - /** - * attempt to add a log record to a dlog. the id and msg should point - * to static strings that are valid throughout the life of the program -diff --git a/src/util/mercury_log.c b/src/util/mercury_log.c -index 483922bf..2fc13548 100644 ---- a/src/util/mercury_log.c -+++ b/src/util/mercury_log.c -@@ -500,16 +500,9 @@ hg_log_outlet_deregister(struct hg_log_outlet *hg_log_outlet) - - if (hg_log_outlet->debug_log && - !(hg_log_outlet->parent && -- hg_log_outlet->parent->debug_log == hg_log_outlet->debug_log)) { -- if (hg_log_outlet->level >= HG_LOG_LEVEL_MIN_DEBUG) { -- FILE *stream = hg_log_streams_g[hg_log_outlet->level] -- ? hg_log_streams_g[hg_log_outlet->level] -- : *hg_log_std_streams_g[hg_log_outlet->level]; -- hg_dlog_dump_counters( -- hg_log_outlet->debug_log, hg_log_func_g, stream, 0); -- } -+ hg_log_outlet->parent->debug_log == hg_log_outlet->debug_log)) - hg_dlog_free(hg_log_outlet->debug_log); -- } -+ - STAILQ_REMOVE(&hg_log_outlets_g, hg_log_outlet, hg_log_outlet, entry); - hg_log_outlet->registered = false; - } -@@ -593,7 +586,8 @@ hg_log_vwrite(struct hg_log_outlet *hg_log_outlet, enum hg_log_level log_level, - no_return ? "" : "\n"); - #endif - -- if (log_level == HG_LOG_LEVEL_ERROR && hg_log_outlet->debug_log && -+ if ((log_level == HG_LOG_LEVEL_ERROR || log_level == HG_LOG_LEVEL_FATAL) && -+ hg_log_outlet->debug_log && - hg_log_outlet->level >= HG_LOG_LEVEL_MIN_DEBUG) { - hg_dlog_dump(hg_log_outlet->debug_log, hg_log_func_g, stream, 0); - hg_dlog_resetlog(hg_log_outlet->debug_log); -diff --git a/src/util/mercury_log.h b/src/util/mercury_log.h -index ceba0c7a..1ed01429 100644 ---- a/src/util/mercury_log.h -+++ b/src/util/mercury_log.h -@@ -308,11 +308,19 @@ - hg_dlog_mkcount32(HG_LOG_OUTLET(name).debug_log, counter_ptr, \ - counter_name, counter_desc) - -+/* HG_LOG_DEL_COUNTER32: delete 32-bit debug log counter */ -+#define HG_LOG_DEL_COUNTER32(name, counter_ptr) \ -+ hg_dlog_rmcount32(HG_LOG_OUTLET(name).debug_log, counter_ptr) -+ - /* HG_LOG_ADD_COUNTER64: add 64-bit debug log counter */ - #define HG_LOG_ADD_COUNTER64(name, counter_ptr, counter_name, counter_desc) \ - hg_dlog_mkcount64(HG_LOG_OUTLET(name).debug_log, counter_ptr, \ - counter_name, counter_desc) - -+/* HG_LOG_DEL_COUNTER64: delete 64-bit debug log counter */ -+#define HG_LOG_DEL_COUNTER64(name, counter_ptr) \ -+ hg_dlog_rmcount64(HG_LOG_OUTLET(name).debug_log, counter_ptr) -+ - /*************************************/ - /* Public Type and Struct Definition */ - /*************************************/ -diff --git a/src/util/version.txt b/src/util/version.txt -index fcdb2e10..ee74734a 100644 ---- a/src/util/version.txt -+++ b/src/util/version.txt -@@ -1 +1 @@ --4.0.0 -+4.1.0 --- -2.52.0 - diff --git a/deps/patches/mercury/0003_combined_plugin_path.patch b/deps/patches/mercury/0003_combined_plugin_path.patch new file mode 100644 index 00000000000..476598dbc84 --- /dev/null +++ b/deps/patches/mercury/0003_combined_plugin_path.patch @@ -0,0 +1,260 @@ +diff --git a/src/na/CMakeLists.txt b/src/na/CMakeLists.txt +index 5c4ec2f..625c07c 100644 +--- a/src/na/CMakeLists.txt ++++ b/src/na/CMakeLists.txt +@@ -82,9 +82,10 @@ if(NA_USE_DYNAMIC_PLUGINS) + if(NOT BUILD_SHARED_LIBS) + message(FATAL_ERROR "Using dynamic plugins requires BUILD_SHARED_LIBS to be ON.") + endif() ++ cmake_path(SET NA_PLUGIN_RELATIVE_PATH ${NA_INSTALL_PLUGIN_DIR}) ++ cmake_path(RELATIVE_PATH NA_PLUGIN_RELATIVE_PATH BASE_DIRECTORY ${NA_INSTALL_LIB_DIR}) ++ message(STATUS "NA plugin install directory: ${NA_INSTALL_PLUGIN_DIR} (relative path to libraries: ${NA_PLUGIN_RELATIVE_PATH})") + set(NA_HAS_DYNAMIC_PLUGINS 1) +- set(NA_DEFAULT_PLUGIN_PATH ${NA_INSTALL_PLUGIN_DIR} CACHE PATH "Default path used to load plugins.") +- mark_as_advanced(NA_DEFAULT_PLUGIN_PATH) + endif() + + # BMI +diff --git a/src/na/na.c b/src/na/na.c +index 781f4c8..b60d305 100644 +--- a/src/na/na.c ++++ b/src/na/na.c +@@ -20,6 +20,9 @@ + # include + # else + # include ++# include ++# include ++# include + # endif + #endif + +@@ -138,6 +141,10 @@ na_plugin_check_protocol(const struct na_class_ops *const class_ops[], + const struct na_class_ops **ops_p); + + #ifdef NA_HAS_DYNAMIC_PLUGINS ++/* Resolve plugin search path */ ++static na_return_t ++na_plugin_resolve_path(const char *offset, char *path, size_t path_size); ++ + /* Scan a given path and return a list of plugins */ + static na_return_t + na_plugin_scan_path(const char *path, struct na_plugin_entry **entries_p); +@@ -265,15 +272,26 @@ static void + na_initialize(void) + { + const char *plugin_path = getenv("NA_PLUGIN_PATH"); ++ char resolved_path[NA_PLUGIN_PATH_MAX]; + na_return_t ret; + +- if (plugin_path == NULL) +- plugin_path = NA_DEFAULT_PLUGIN_PATH; ++ if (plugin_path == NULL) { ++ ret = na_plugin_resolve_path( ++ NA_PLUGIN_RELATIVE_PATH, resolved_path, sizeof(resolved_path)); ++ NA_CHECK_SUBSYS_NA_ERROR(cls, done, ret, ++ "Could not resolve plugin path using offset (%s)", ++ NA_PLUGIN_RELATIVE_PATH); ++ plugin_path = resolved_path; ++ } + + ret = na_plugin_scan_path(plugin_path, &na_plugin_dynamic_g); + NA_CHECK_SUBSYS_WARNING(fatal, ret != NA_SUCCESS, +- "No plugin found in path (%s), consider setting NA_PLUGIN_PATH.", ++ "No usable plugin found in path (%s), consider setting NA_PLUGIN_PATH " ++ "if path indicated is not valid.", + plugin_path); ++ ++done: ++ return; + } + + /*---------------------------------------------------------------------------*/ +@@ -472,6 +490,44 @@ error: + /*---------------------------------------------------------------------------*/ + #ifdef NA_HAS_DYNAMIC_PLUGINS + # ifdef _WIN32 ++# define PATH_MAX MAX_PATH ++# define realpath(N, R) _fullpath((R), (N), PATH_MAX) ++# endif ++static na_return_t ++na_plugin_resolve_path(const char *offset, char *path, size_t path_size) ++{ ++ static int placeholder; ++ char libpath[PATH_MAX]; ++ char *slash; ++ na_return_t ret; ++ int rc; ++ ++ rc = hg_dl_get_path(&placeholder, path, path_size); ++ NA_CHECK_SUBSYS_ERROR( ++ cls, rc != 0, error, ret, NA_NOENTRY, "hg_dl_get_path() failed"); ++ ++ NA_CHECK_SUBSYS_ERROR(cls, realpath(path, libpath) == NULL, error, ret, ++ NA_NOENTRY, "realpath() failed, %s", strerror(errno)); ++ ++ slash = strrchr(libpath, '/'); ++ NA_CHECK_SUBSYS_ERROR(cls, slash == NULL, error, ret, NA_INVALID_ARG, ++ "Could not find last '/' in %s", libpath); ++ *slash = '\0'; ++ ++ rc = snprintf(path, path_size, "%s/%s", libpath, offset); ++ NA_CHECK_SUBSYS_ERROR(cls, rc < 0 || rc > (int) path_size, error, ret, ++ NA_OVERFLOW, ++ "snprintf() failed or name truncated, rc: %d (expected %zu)", rc, ++ path_size); ++ ++ return NA_SUCCESS; ++ ++error: ++ return ret; ++} ++ ++/*---------------------------------------------------------------------------*/ ++# ifdef _WIN32 + static na_return_t + na_plugin_scan_path(const char *path, struct na_plugin_entry **entries_p) + { +@@ -494,7 +550,7 @@ na_plugin_scan_path(const char *path, struct na_plugin_entry **entries_p) + struct dirent **plugin_list; + struct na_plugin_entry *entries = NULL; + na_return_t ret; +- int n, n_entries = 0; ++ int n, opened_plugins = 0; + + n = scandir(path, &plugin_list, na_plugin_filter, alphasort); + NA_CHECK_SUBSYS_ERROR( +@@ -504,16 +560,20 @@ na_plugin_scan_path(const char *path, struct na_plugin_entry **entries_p) + (struct na_plugin_entry *) calloc((size_t) n + 1, sizeof(*entries)); + NA_CHECK_SUBSYS_ERROR(cls, entries == NULL, error, ret, NA_NOMEM, + "Could not allocate %d plugin entries", n); +- n_entries = n; + + while (n--) { + ret = na_plugin_open(path, plugin_list[n]->d_name, &entries[n]); + free(plugin_list[n]); +- NA_CHECK_SUBSYS_NA_ERROR(cls, error, ret, "Could not open plugin (%s)", +- plugin_list[n]->d_name); ++ if (ret == NA_SUCCESS) ++ opened_plugins++; ++ else ++ NA_CHECK_SUBSYS_NA_ERROR( ++ cls, error, ret, "Could not open plugin (%s)", plugin_list[n]->d_name); + } + + free(plugin_list); ++ NA_CHECK_SUBSYS_ERROR(cls, opened_plugins == 0, error, ret, NA_NOENTRY, ++ "No usable plugin found in path (%s)", path); + + *entries_p = entries; + +@@ -521,19 +581,11 @@ na_plugin_scan_path(const char *path, struct na_plugin_entry **entries_p) + + error: + if (n > 0) { +- if (entries != NULL) { +- int i; +- +- /* close entry */ +- for (i = n + 1; i < n_entries; i++) +- na_plugin_close(&entries[i]); +- free(entries); +- } +- + while (n--) + free(plugin_list[n]); + free(plugin_list); + } ++ free(entries); + + return ret; + } +diff --git a/src/na/na_config.h.in b/src/na/na_config.h.in +index 30d0e08..50dd443 100644 +--- a/src/na/na_config.h.in ++++ b/src/na/na_config.h.in +@@ -80,7 +80,7 @@ + # define NA_PLUGIN + # define NA_PLUGIN_VISIBILITY NA_PRIVATE + #endif +-#cmakedefine NA_DEFAULT_PLUGIN_PATH "@NA_DEFAULT_PLUGIN_PATH@" ++#cmakedefine NA_PLUGIN_RELATIVE_PATH "@NA_PLUGIN_RELATIVE_PATH@" + + /* Build Options */ + #cmakedefine NA_HAS_DEBUG +diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt +index 44fe6b9..32f51d1 100644 +--- a/src/util/CMakeLists.txt ++++ b/src/util/CMakeLists.txt +@@ -193,6 +193,7 @@ configure_file( + #------------------------------------------------------------------------------ + set(MERCURY_UTIL_SRCS + ${CMAKE_CURRENT_SOURCE_DIR}/mercury_atomic_queue.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/mercury_dl.c + ${CMAKE_CURRENT_SOURCE_DIR}/mercury_dlog.c + ${CMAKE_CURRENT_SOURCE_DIR}/mercury_event.c + ${CMAKE_CURRENT_SOURCE_DIR}/mercury_hash_table.c +diff --git a/src/util/mercury_dl.h b/src/util/mercury_dl.h +index b86932a..0cda094 100644 +--- a/src/util/mercury_dl.h ++++ b/src/util/mercury_dl.h +@@ -62,6 +62,18 @@ hg_dl_close(HG_DL_HANDLE handle); + static HG_UTIL_INLINE void * + hg_dl_sym(HG_DL_HANDLE handle, const char *name); + ++/** ++ * Retrieve library path. ++ * ++ * \param addr [IN] address of the symbol ++ * \param path [OUT] buffer to store the path ++ * \param path_size [IN] size of the buffer ++ * ++ * \return Non-negative on success or negative on failure ++ */ ++HG_UTIL_PUBLIC int ++hg_dl_get_path(const void *addr, char *path, size_t path_size); ++ + /*---------------------------------------------------------------------------*/ + static HG_UTIL_INLINE const char * + hg_dl_error(void) +diff --git a/src/util/mercury_dl.c b/src/util/mercury_dl.c +new file mode 100644 +index 0000000..6ed4666 +--- /dev/null ++++ b/src/util/mercury_dl.c +@@ -0,0 +1,35 @@ ++/** ++ * Copyright (c) 2013-2022 UChicago Argonne, LLC and The HDF Group. ++ * Copyright (c) 2022-2023 Intel Corporation. ++ * ++ * SPDX-License-Identifier: BSD-3-Clause ++ */ ++ ++#if !defined(_WIN32) && !defined(_GNU_SOURCE) ++# define _GNU_SOURCE ++#endif ++#include "mercury_dl.h" ++ ++#include ++ ++/*---------------------------------------------------------------------------*/ ++int ++hg_dl_get_path(const void *addr, char *path, size_t path_size) ++{ ++#ifdef _WIN32 ++ HMODULE module; ++ if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | ++ GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, ++ (LPCSTR) addr, &module)) ++ return GetModuleFileNameA(module, path, (DWORD) path_size); ++#else ++ Dl_info info; ++ if (dladdr(addr, &info) && info.dli_fname) { ++ strncpy(path, info.dli_fname, path_size); ++ path[path_size - 1] = '\0'; ++ return HG_UTIL_SUCCESS; ++ } ++#endif ++ ++ return HG_UTIL_FAIL; ++} diff --git a/deps/patches/mercury/0003_ofi_auth_key.patch b/deps/patches/mercury/0003_ofi_auth_key.patch deleted file mode 100644 index 480f6518dd9..00000000000 --- a/deps/patches/mercury/0003_ofi_auth_key.patch +++ /dev/null @@ -1,25 +0,0 @@ -From cd678a20fb21b6e5a5b8f05619427dc79aa1246b Mon Sep 17 00:00:00 2001 -From: Jerome Soumagne -Date: Mon, 26 Jan 2026 12:27:27 -0600 -Subject: [PATCH 3/3] NA OFI: ensure domain auth key remains valid - ---- - src/na/na_ofi.c | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/src/na/na_ofi.c b/src/na/na_ofi.c -index d385d279..68a443e8 100644 ---- a/src/na/na_ofi.c -+++ b/src/na/na_ofi.c -@@ -5220,7 +5220,7 @@ na_ofi_domain_open(const struct na_ofi_fabric *na_ofi_fabric, - NA_NOMEM, "Could not allocate auth_key"); - memcpy(na_ofi_domain->auth_key, &base_auth_key, sizeof(base_auth_key)); - -- domain_attr->auth_key = (void *) &base_auth_key; -+ domain_attr->auth_key = (void *) na_ofi_domain->auth_key; - domain_attr->auth_key_size = auth_key_size; - } - --- -2.52.0 - diff --git a/docs/admin/hardware.md b/docs/admin/hardware.md index c35b00af025..3f4658cf40b 100644 --- a/docs/admin/hardware.md +++ b/docs/admin/hardware.md @@ -35,10 +35,9 @@ validated on a regular basis. An RDMA-capable fabric is preferred for best performance. The DAOS data plane relies on [OFI libfabric](https://ofiwg.github.io/libfabric/) and supports OFI providers for Ethernet/tcp and InfiniBand/verbs. -[UCX](https://www.openucx.org/) -is also supported as an alternative network stack for DAOS on InfiniBand/verbs -platforms. -Refer to [UCX Fabric Support](./ucx.md) +Starting with a Technology Preview in DAOS 2.2, [UCX](https://www.openucx.org/) +is also supported as an alternative network stack for DAOS. +Refer to [UCX Fabric Support (DAOS 2.2 Technology Preview)](./ucx.md) for details on setting up DAOS with UCX support. DAOS supports multiple network interfaces on the servers diff --git a/docs/admin/ucx.md b/docs/admin/ucx.md index c366fb1cf06..b6529c7f1ee 100644 --- a/docs/admin/ucx.md +++ b/docs/admin/ucx.md @@ -16,26 +16,33 @@ the following steps are needed: for information about supported MLNX\_OFED releases. * The `mercury-ucx` RPM package needs to be **manually** selected for - installation. The base `mercury` RPM package ships with no plugins. - The `mercury-ucx` RPM contains the UCX plugin that is required for - enabling UCX support. - This RPM **must** be used in InfiniBand environments when the intention - is to use UCX. Attempts to install this RPM in non-Infiniband environments - will fail, because it has a dependency on UCX packages. + installation: + + - The base `mercury` RPM package ships with the libfabric plugin. + This RPM will be installed by default and is a dependency of the + `mercury-ucx` RPM. + + - The additional `mercury-ucx` RPM is also provided. This RPM contains + the UCX plugin that is required for enabling UCX support. + This RPM **must** be used in + InfiniBand environments when the intention is to use + UCX. + Attempts to install this RPM in non-Infiniband environments + will fail, because it has a dependency on UCX packages. * At DAOS **installation** time, to enable UCX support the `mercury-ucx` RPM package must be explicitly listed. - For example, using the `dnf`/`yum` package manager on EL8: + For example, using the `yum`/`dnf` package manager on EL8: ```bash # on DAOS_ADMIN nodes: - dnf install mercury-ucx daos-admin + yum install mercury-ucx daos-admin # on DAOS_SERVER nodes: - dnf install mercury-ucx daos-server + yum install mercury-ucx daos-server # on DAOS_CLIENT nodes: - dnf install mercury-ucx daos-client + yum install mercury-ucx daos-client ``` After UCX support has been enabled by installing the `mercury-ucx` diff --git a/site_scons/components/__init__.py b/site_scons/components/__init__.py index f6ab99d6245..da2518a9928 100644 --- a/site_scons/components/__init__.py +++ b/site_scons/components/__init__.py @@ -1,6 +1,6 @@ # Copyright 2016-2024 Intel Corporation # Copyright 2025 Google LLC -# Copyright 2025-2026 Hewlett Packard Enterprise Development LP +# Copyright 2025 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -202,13 +202,17 @@ def define_mercury(reqs): '-DMERCURY_USE_SYSTEM_BOOST:BOOL=ON', '-DMERCURY_USE_CHECKSUMS:BOOL=OFF', '-DMERCURY_ENABLE_COUNTERS:BOOL=ON', - '-DMERCURY_ENABLE_DEBUG:BOOL=ON', '-DNA_USE_DYNAMIC_PLUGINS:BOOL=ON', '-DNA_USE_SM:BOOL=ON', '-DNA_USE_OFI:BOOL=ON', '-DNA_USE_UCX:BOOL=ON', '../mercury'] + if reqs.target_type == 'debug': + mercury_build.append('-DMERCURY_ENABLE_DEBUG:BOOL=ON') + else: + mercury_build.append('-DMERCURY_ENABLE_DEBUG:BOOL=OFF') + reqs.define('mercury', retriever=GitRepoRetriever(True), commands=[mercury_build, diff --git a/src/cart/utils/memcheck-cart.supp b/src/cart/utils/memcheck-cart.supp index a2f73cfcdad..9676c1966f2 100644 --- a/src/cart/utils/memcheck-cart.supp +++ b/src/cart/utils/memcheck-cart.supp @@ -512,14 +512,6 @@ fun:hg_dlog_mkcount32 ... } -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:hg_dlog_mkcount64 - ... -} { FI leak 9 Memcheck:Leak @@ -535,6 +527,18 @@ fun:HG_Init_opt fun:crt_hg_class_init } +{ + Tcp provider + Memcheck:Param + sendmsg(msg.msg_iov[1]) + ... + fun:sendmsg + fun:ofi_sockapi_sendv_socket + fun:ofi_bsock_sendv + ... + fun:fi_senddata + ... +} { Tcp provider with ofi rxm Memcheck:Param @@ -542,14 +546,20 @@ ... fun:ofi_bsock_sendv ... + fun:fi_tsend + ... } { Tcp provider with ofi rxm 2 Memcheck:Param sendmsg(msg.msg_iov[2]) ... + fun:sendmsg + fun:ofi_sockapi_sendv_socket fun:ofi_bsock_sendv ... + fun:fi_tsend + ... } { Go syscall. @@ -767,6 +777,24 @@ fun:__tsan_go_atomic64_fetch_add fun:racecall } +{ + DAOS-17006 - mercury leak + Memcheck:Leak + match-leak-kinds: reachable + fun:malloc + fun:hg_dlog_mkcount64 + fun:hg_core_counters_init + fun:hg_core_init + fun:HG_Core_init_opt2 + fun:HG_Init_opt2 + fun:crt_hg_class_init + fun:crt_hg_ctx_init + fun:crt_context_provider_create + fun:daos_eq_lib_init + fun:daos_init + fun:_cgo_b590e4e2531a_Cfunc_daos_init + fun:runtime.asmcgocall.abi0 +} { getpwnam_r() leak Memcheck:Leak diff --git a/utils/build.config b/utils/build.config index 42b49fcd125..c36fbd2a043 100644 --- a/utils/build.config +++ b/utils/build.config @@ -9,7 +9,7 @@ isal=v2.31.1 isal_crypto=v2.25.0 spdk=v24.09 ofi=v1.22.0 -mercury=v2.4.1 +mercury=v2.4.0 protobufc=v1.3.3 ucx=v1.14.1 @@ -27,5 +27,5 @@ ucx=https://github.com/openucx/ucx.git [patch_versions] spdk=0001_3428322b812fe31cc3e1d0308a7f5bd4b06b9886.diff,0002_spdk_rwf_nowait.patch,0003_external_isal.patch -mercury=0001_dep_versions.patch,0002_ofi_counters.patch,0003_ofi_auth_key.patch +mercury=0001_na_ucx.patch,0002_na_ucx_ep_flush.patch,0003_combined_plugin_path.patch argobots=0001_411e5b344642ebc82190fd8b125db512e5b449d1.diff,0002_bb0c908abfac4bfe37852eee621930634183c6aa.diff diff --git a/utils/rpms/daos.changelog b/utils/rpms/daos.changelog index 3a0a53f5251..40e3d18fde6 100644 --- a/utils/rpms/daos.changelog +++ b/utils/rpms/daos.changelog @@ -1,8 +1,4 @@ %changelog -* Fri Jan 16 2026 Jerome Soumagne 2.7.103-2 -- Drop libfabric-devel build requirement -- Drop libfabric requirement that is already provided by mercury-libfabric - * Fri Dec 19 2025 Dalton Bohning 2.7.103-1 - Bump version to 2.7.103 diff --git a/utils/rpms/daos.sh b/utils/rpms/daos.sh index 1171a079746..8a28b2a187d 100755 --- a/utils/rpms/daos.sh +++ b/utils/rpms/daos.sh @@ -1,10 +1,6 @@ #!/bin/bash -# -# (C) Copyright 2025 Google LLC -# Copyright 2025-2026 Hewlett Packard Enterprise Development LP -# -# SPDX-License-Identifier: BSD-2-Clause-Patent -# +# (C) Copyright 2025 Google LLC +# WORK IN PROGRESS set -eEuo pipefail root="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" . "${root}/fpm_common.sh" @@ -68,7 +64,7 @@ install_list+=("${tmp}${sysconfdir}/daos/certs=${sysconfdir}/daos") EXTRA_OPTS+=("--rpm-attr" "0755,root,root:${sysconfdir}/daos/certs") -DEPENDS=( "mercury >= ${mercury_version}" ) +DEPENDS=( "mercury >= ${mercury_full}" "${libfabric_lib} >= ${libfabric_full}" ) DEPENDS+=( "${isal_crypto_lib} >= ${isal_crypto_version}" ) build_package "daos" diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 2a1827b0176..a8f5da38d1f 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -12,6 +12,7 @@ %global daos_build_args client test %endif %global mercury_version 2.4 +%global libfabric_version 1.15.1-1 %global argobots_version 1.2 %global __python %{__python3} %global daos_log_dir "/var/log/daos" @@ -24,7 +25,7 @@ Name: daos Version: 2.7.103 -Release: 2%{?relval}%{?dist} +Release: 1%{?relval}%{?dist} Summary: DAOS Storage Engine License: BSD-2-Clause-Patent @@ -39,6 +40,7 @@ BuildRequires: python3-scons >= 2.4 %else BuildRequires: scons >= 2.4 %endif +BuildRequires: libfabric-devel >= %{libfabric_version} BuildRequires: mercury-devel >= %{mercury_version} BuildRequires: gcc-c++ %if (0%{?rhel} >= 8) @@ -165,10 +167,12 @@ Requires: ndctl %if (0%{?suse_version} >= 1500) Requires: ipmctl >= 03.00.00.0423 Requires: libpmemobj1 >= 2.1.0-1.suse1500 +Requires: libfabric1 >= %{libfabric_version} %else Requires: ipmctl >= 03.00.00.0468 Requires: libpmemobj >= 2.1.0-1%{?dist} %endif +Requires: libfabric >= %{libfabric_version} Requires: mercury >= %{mercury_version} Requires(post): /sbin/ldconfig Requires(postun): /sbin/ldconfig @@ -191,6 +195,10 @@ This package contains DAOS administrative tools (e.g. dmg). Summary: The DAOS client Requires: %{name}%{?_isa} = %{version}-%{release} Requires: mercury >= %{mercury_version} +Requires: libfabric >= %{libfabric_version} +%if (0%{?suse_version} >= 1500) +Requires: libfabric1 >= %{libfabric_version} +%endif Requires: /usr/bin/fusermount3 %{?systemd_requires} diff --git a/utils/rpms/mercury.changelog b/utils/rpms/mercury.changelog deleted file mode 100644 index c00f682eb61..00000000000 --- a/utils/rpms/mercury.changelog +++ /dev/null @@ -1,274 +0,0 @@ -%changelog -* Mon Jan 26 2026 Jerome Soumagne - 2.4.1-1 -- Update to 2.4.1 -- Separate libfabric plugin from main build to align with ucx plugin -- Add patches for runtime version checks and libfabric plugin counters -- Add patch for libfabric auth key - -* Wed Jun 25 2025 Joseph Moore - 2.4.0-5 -- Update release number to differentiate from test RPMs for prior issue. - -* Tue Mar 11 2025 Joseph Moore - 2.4.0-4 -- Change to addr_release for handling of "already present" warning. - -* Wed Jan 15 2025 Joseph Moore - 2.4.0-3 -- Add patch to na_ucx.c to flush end point prior to close. - -* Tue Jan 07 2025 Joseph Moore - 2.4.0-2 -- Enable debug RPMs for Leap sub-packages. - -* Mon Nov 04 2024 Jerome Soumagne - 2.4.0-1 -- Update to 2.4.0 -- Update required libfabric version (>= 1.20) - -* Mon Oct 07 2024 Joseph Moore - 2.4.0~rc5-5 -- Update patch to na_ucx.c to set thread-safe on clients. - -* Thu Sep 26 2024 Joseph Moore - 2.4.0~rc5-4 -- Update patch to na_ucx.c to add fix for connection accept. - -* Wed Sep 04 2024 Brian J. Murrell - 2.4.0~rc5-3 -- Add --without ucx build switch - -* Thu Aug 29 2024 Joseph Moore - 2.4.0~rc5-2 -- Add patch to na_ucx.c to check ep in key_resolve. - -* Mon Aug 26 2024 Jerome Soumagne - 2.4.0~rc5-1 -- Update to 2.4.0rc5 - -* Fri Aug 02 2024 Jerome Soumagne - 2.4.0~rc4-1 -- Update to 2.4.0rc4 -- Remove previous patches now included in 2.4 -- Require libfabric >= 1.15 - -* Tue Mar 19 2024 Jerome Soumagne - 2.3.1-3 -- Add patch to fix ucx hg_info -- Add patch to remove ofi cxi MR warnings -- Add patch to fix potential segfault on log free - -* Wed Nov 22 2023 Jerome Soumagne - 2.3.1-2 -- Rebuild for EL 8.8 and Leap 15.5 - -* Fri Oct 27 2023 Jerome Soumagne - 2.3.1-1 -- Update to 2.3.1 -- Add json-c dependency for hg_info JSON output support -- Drop support for CentOS7 - -* Tue Sep 26 2023 Joseph Moore - 2.3.1~rc1-2 -- Add patch to na_ucx.c to force retry of out-of-memory error. - -* Tue Aug 29 2023 Jerome Soumagne - 2.3.1~rc1-1 -- Update to 2.3.1rc1 - -* Thu Jun 22 2023 Brian J. Murrell - 2.3.0-2 -- Rebuild for EL9 - -* Wed Jun 7 2023 Jerome Soumagne - 2.3.0-1 -- Update to 2.3.0 -- Add hg_info tool -- Fix pie flags on CentOS7 -- Remove na_ucx_src_port.patch and old patches - -* Tue Apr 25 2023 Jerome Soumagne - 2.3.0~rc5-1 -- Update to 2.3.0rc5 -- Remove na_ucx.c patch and add temporary na_ucx_src_port.patch -- Update build to make use of NA dynamic plugins -- Fix source URL and package perf tests - -* Thu Dec 22 2022 Joseph Moore - 2.2.0-6 -- Regenerate packages for LEAP15.4 - -* Thu Nov 17 2022 Joseph Moore - 2.2.0-5 -- Update na_ucx.c patch to support reconnection following a disconnect. - -* Wed Oct 05 2022 Joseph Moore - 2.2.0-4 -- Update na_ucx.c patch to include UCX status to NA error mapping. - -* Tue Sep 20 2022 Joseph Moore - 2.2.0-3 -- Fix defect in connect function. - -* Fri Sep 09 2022 Joseph Moore - 2.2.0-2 -- Add na_ucx.c patch to change ep creation for single IB device. - -* Fri Aug 5 2022 Jerome Soumagne - 2.2.0-1 -- Update to 2.2.0 - -* Mon Aug 1 2022 Jerome Soumagne - 2.2.0~rc6-2 -- Rebuild after libfabric rpm dropped CXI compat patch -- Drop CXI compat patch - -* Mon Jun 27 2022 Jerome Soumagne - 2.2.0~rc6-1 -- Update to 2.2.0rc6 -- Skip install rpath, enable debug log. -- Remove openpa dependency. - -* Fri Apr 22 2022 Joseph Moore - 2.1.0~rc4-9 -- Change ucx unified mode to off (updated UCX patch file). - -* Fri Apr 1 2022 Brian J. Murrell - 2.1.0~rc4-8 -- Build with ucx subpackage on supported platforms -- Removed invalid build options: - * MERCURY_ENABLE_VERBOSE_ERROR - * MERCURY_USE_SELF_FORWARD - -* Thu Mar 31 2022 Joseph Moore - 2.1.0~rc4-7 -- Apply daos-9679 address parsing change and active message revision to na_ucx.c. - -* Fri Mar 11 2022 Alexander Oganezov - 2.1.0~rc4-6 -- Apply cxi provider patch - -* Tue Feb 22 2022 Alexander Oganezov - 2.1.0~rc4-5 -- Apply doas-9561 workaround - -* Thu Feb 17 2022 Brian J. Murrell - 2.1.0~rc4-4 -- Fix issues with %%post* ldconfig - - No lines are allowed after %%post -p - - These are not needed on EL8 as it's glibc does the work - -* Thu Dec 23 2021 Alexander Oganezov - 2.1.0~rc4-3 -- Remove daos-9173 workaround -- Apply cpu usage fix to mercury - -* Tue Dec 7 2021 Alexander Oganezov - 2.1.0~rc4-2 -- Apply DAOS-9173 workaround patch to na_ofi.c - -* Tue Nov 30 2021 Alexander Oganezov - 2.1.0~rc4-1 -- Update to version v2.1.0rc4 - -* Tue Oct 12 2021 Alexander Oganezov - 2.1.0~rc2-1 -- Update to version v2.1.0rc2 - -* Fri May 14 2021 Alexander Oganezov - 2.0.1-1 -- Update to version v2.0.1 - -* Mon May 10 2021 Brian J. Murryyell - 2.0.1~rc1-2 -- Enable debuginfo package building for SUSE - -* Wed Jan 20 2021 Alexander Oganezov - 2.0.1~rc1-1 -- Update to version v2.0.1rc1 - -* Wed Nov 18 2020 Alexander Oganezov - 2.0.0-1 -- Update to release v2.0.0 - -* Wed Oct 28 2020 Alexander Oganezov - 2.0.0~rc3-1 -- Update to release v2.0.0rc3 - -* Mon Oct 12 2020 Alexander Oganezov - 2.0.0~rc2-1 -- Update to release v2.0.0rc2 - -* Tue Aug 18 2020 Brian J. Murryyell - 2.0.0~rc1-2 -- Use release tarball and not individual submodule tarballs - -* Mon Jul 6 2020 Alexander A Oganezov - 2.0.0~rc1-1 -- Update to release v2.0.0rc1 - -* Mon Jun 22 2020 Brian J. Murryyell - 2.0.0~a1-2 -- Fix License: -- Add %%license - -* Thu May 07 2020 Brian J. Murrell - 2.0.0~a1-1 -- Fix pre-release tag in Version: -- Add Requires: libfabric-devel to devel package - -* Thu Apr 9 2020 Alexander A Oganezov - 2.0.0a1-0.8 -- Update to 4871023058887444d47ead4d089c99db979f3d93 - -* Tue Mar 17 2020 Alexander A Oganezov - 2.0.0a1-0.7 -- Update to 41caa143a07ed179a3149cac4af0dc7aa3f946fd - -* Thu Mar 12 2020 Alexander A Oganezov - 2.0.0a1-0.6 -- Update to 299b06d47e6c1d59a45985dcbbebe3caca0189d0 - -* Tue Mar 10 2020 Alexander A Oganezov - 2.0.0a1-0.5 -- Updated to ad5a3b3dbf171a97e1ca5f1683299db1c69b03ea - -* Thu Mar 05 2020 Vikram Chhabra - 2.0.0a1-0.4 -- Updated to latest master with HG_Forward fix. - -* Tue Feb 11 2020 Yulu Jia - 2.0.0a1-0.3 -- Remove nameserver patch - -* Sun Feb 09 2020 Yulu Jia - 2.0.0a1-0.2 -- Update patch to enable ip:port URI format for psm2 - -* Tue Feb 04 2020 Brian J. Murrell - 2.0.0a1-0.1 -- Update to 2.0.0a1 - -* Tue Jan 28 2020 Yulu Jia - 1.0.1-22 -- Update to c2c2628 -- Apply patch to enable ip:port URI format for psm2 - -* Mon Dec 2 2019 Alexander Oganezov - 1.0.1-21 -- Removed sl_patch on top of 7b529b -- Updated to 9889a0 - -* Thu Oct 31 2019 Alexander Oganezov - 1.0.1-20 -- sl_patch on top of 7b529b - -* Wed Oct 23 2019 Alexander Oganezov - 1.0.1-19 -- Update to 7b529b - -* Tue Oct 22 2019 Alexander Oganezov - 1.0.1-18 -- Reverting from 6a8b693 due to mercury segfaults - -* Mon Oct 21 2019 Alexander Oganezov - 1.0.1-17 -- Update to 6a8b693 - -* Wed Oct 16 2019 Alexander Oganezov - 1.0.1-16 -- Fixed spec to apply patch for 616fee properly - -* Tue Oct 15 2019 Alexander Oganezov - 1.0.1-15 -- Update to 616fee to get latest changes - -* Wed Oct 02 2019 Brian J. Murrell - 1.0.1-14 -- Update to cc0807 to include the HG_Cancel() fix. -- Update to f0b9f9 to get latest changes - -* Wed Oct 02 2019 Brian J. Murrell - 1.0.1-13 -- Once again revert previous update - -* Wed Oct 02 2019 Brian J. Murrell - 1.0.1-12 -- Update to cc0807 to include the HG_Cancel() fix. -- Update to f0b9f9 to get latest changes - -* Wed Sep 25 2019 Brian J. Murrell - 1.0.1-11 -- Back out previous update - - not all consumers are ready for it yet so they need to - pin their BR - -* Fri Sep 20 2019 Brian J. Murrell - 1.0.1-10 -- Update to cc0807 to include the HG_Cancel() fix. -- Update to f0b9f9 to get latest changes - -* Thu Aug 08 2019 Brian J. Murrell - 1.0.1-9 -- Revert previous update - -* Fri Aug 02 2019 Yulu Jia - 1.0.1-8 -- Update to cc0807 to include the HG_Cancel() fix. -- Roll the version number back to 1.0.1 - -* Fri Aug 02 2019 Brian J. Murrell - 1.0.1-7 -- Revert back to the 1.0.1-4 release as the upgrade included - in -5 (and the subsequent fix in -6) was premature - -* Thu Aug 01 2019 Brian J. Murrell - 1.0.1-6 -- Roll the version number back to 1.0.1 - -* Fri Jul 26 2019 Yulu Jia - 1.0.1-5 -- Update to cc0807 to include the HG_Cancel() fix. - -* Thu May 02 2019 Brian J. Murrell - 1.0.1-4 -- Devel package needs to require the lib package - -* Fri Mar 15 2019 Brian J. Murrell - 1.0.1-2 -- Add patch to revert back to Dec 06, 2018 c68870f - -* Mon Mar 11 2019 Brian J. Murrell - 1.0.1-1 -- Update to 1.0.1 -- Add patch for "HG Core: fix missing static inline in mercury_core.h" - -* Wed Oct 24 2018 Brian J. Murrell - 0.9.0-1.git.0f8f25b -- Update mercury to git sha1 0f8f25bb3d57f117979de65cc3c05cf192cf4b31 - -* Mon Aug 20 2018 Brian J. Murrell - 0.9.0-1.git.f7f6955 -- Initial package diff --git a/utils/rpms/mercury.sh b/utils/rpms/mercury.sh index 5f30dd89372..f14fe057043 100755 --- a/utils/rpms/mercury.sh +++ b/utils/rpms/mercury.sh @@ -1,10 +1,5 @@ #!/bin/bash -# -# (C) Copyright 2025 Google LLC -# Copyright 2025-2026 Hewlett Packard Enterprise Development LP -# -# SPDX-License-Identifier: BSD-2-Clause-Patent -# +# (C) Copyright 2025 Google LLC set -eEuo pipefail root="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" . "${root}/fpm_common.sh" @@ -29,7 +24,6 @@ Access (RMA). Its interface is generic and allows any function call to be serialized. Since code generation is done using the C preprocessor, no external tool is required." URL="http://mercury-hpc.github.io" -RPM_CHANGELOG="mercury.changelog" files=() TARGET_PATH="${bindir}" @@ -42,17 +36,14 @@ list_files files "${SL_MERCURY_PREFIX}/lib64/lib*.so.*" clean_bin "${files[@]}" append_install_list "${files[@]}" -ARCH="${isa}" -build_package "mercury" - TARGET_PATH="${libdir}/mercury" list_files files "${SL_MERCURY_PREFIX}/lib64/mercury/libna_plugin_ofi.so" clean_bin "${files[@]}" append_install_list "${files[@]}" ARCH="${isa}" -DEPENDS=("${libfabric_lib} >= ${libfabric_min_version}") -build_package "mercury-libfabric" +DEPENDS=("${libfabric_lib} >= ${libfabric_version}") +build_package "mercury" DEPENDS=() TARGET_PATH="${libdir}/mercury" diff --git a/utils/rpms/package_info.sh b/utils/rpms/package_info.sh index ac6356c3c72..cc3be377607 100644 --- a/utils/rpms/package_info.sh +++ b/utils/rpms/package_info.sh @@ -1,10 +1,5 @@ #!/bin/bash -# -# (C) Copyright 2025 Google LLC -# Copyright 2025-2026 Hewlett Packard Enterprise Development LP -# -# SPDX-License-Identifier: BSD-2-Clause-Patent -# +# (C) Copyright 2025 Google LLC root="$(realpath "$(dirname "$(dirname "$(dirname "${BASH_SOURCE[0]}")")")")" set_lib_name() { comp="$1"; shift @@ -40,12 +35,11 @@ daos_release="$(grep "^Release: " "${root}/utils/rpms/daos.spec" | \ sed 's/^Release: *//' | sed 's/%.*//')${DAOS_RELVAL:-}${distro_name}" export daos_release -export libfabric_min_version="1.20" export libfabric_version="1.22.0" export libfabric_release="5${distro_name}" export libfabric_full="${libfabric_version}-${libfabric_release}" -export mercury_version="2.4.1" -export mercury_release="1${distro_name}" +export mercury_version="2.4.0" +export mercury_release="8${distro_name}" export mercury_full="${mercury_version}-${mercury_release}" export argobots_version="1.2" export argobots_release="4${distro_name}" @@ -100,8 +94,6 @@ set_lib_name mercury dev mercury mercury mercury export mercury_dev set_lib_name mercury lib mercury mercury mercury export mercury_lib -set_lib_name mercury_libfabric lib mercury-libfabric mercury-libfabric mercury-libfabric -export mercury_libfabric_lib set_lib_name pmemobj lib libpmemobj libpmemobj1 libpmemobj1 set_lib_name pmemobj dev libpmemobj libpmemobj1 libpmemobj1 diff --git a/utils/test_memcheck.supp b/utils/test_memcheck.supp index ef69713271a..4f5e2bca077 100644 --- a/utils/test_memcheck.supp +++ b/utils/test_memcheck.supp @@ -271,11 +271,15 @@ ... } { - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:hg_dlog_mkcount64 + Tcp provider + Memcheck:Param + sendmsg(msg.msg_iov[1]) + ... + fun:sendmsg + fun:ofi_sockapi_sendv_socket + fun:ofi_bsock_sendv + ... + fun:fi_senddata ... } { @@ -285,14 +289,20 @@ ... fun:ofi_bsock_sendv ... + fun:fi_tsend + ... } { Tcp provider with ofi rxm 2 Memcheck:Param sendmsg(msg.msg_iov[2]) ... + fun:sendmsg + fun:ofi_sockapi_sendv_socket fun:ofi_bsock_sendv ... + fun:fi_tsend + ... } { par_init mpi or dlopen leak