diff --git a/CMakeLists.txt b/CMakeLists.txt index 0606e29c3..a2a5cb6b0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -731,6 +731,7 @@ add_subdirectory(providers/efa/man) add_subdirectory(providers/erdma) add_subdirectory(providers/hns) add_subdirectory(providers/hns/man) +add_subdirectory(providers/ionic) add_subdirectory(providers/irdma) add_subdirectory(providers/mana) add_subdirectory(providers/mana/man) diff --git a/MAINTAINERS b/MAINTAINERS index 2a5783b86..ed1dc0d6f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -77,6 +77,13 @@ M: Chengchang Tang S: Supported F: providers/hns/ +IONIC USERSPACE PROVIDER (for ionic_rdma.ko) +M: Allen Hubbe +M: Andrew Boyer +M: Abhijit Gangurde +S: Supported +F: providers/ionic/ + IRDMA USERSPACE PROVIDER (for i40iw.ko and irdma.ko) M: Sindhu Devale M: Tatyana Nikolova diff --git a/README.md b/README.md index ddd9212ad..2a0c2737d 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ is included: - iw_cxgb4.ko - hfi1.ko - hns-roce-hw-v2.ko + - ionic_rdma.ko - irdma.ko - ib_qib.ko - mana_ib.ko diff --git a/debian/control b/debian/control index 3daf383e7..c714d1689 100644 --- a/debian/control +++ b/debian/control @@ -88,6 +88,7 @@ Description: User space provider drivers for libibverbs - erdma: Alibaba Elastic RDMA (iWarp) Adapter - hfi1verbs: Intel Omni-Path HFI - hns: HiSilicon Hip06 SoC + - ionic: AMD Pensando Distributed Services Card (DSC) RDMA/RoCE Support - ipathverbs: QLogic InfiniPath HCAs - irdma: Intel Ethernet Connection RDMA - mana: Microsoft Azure Network Adapter diff --git a/debian/copyright b/debian/copyright index f4e1b8b4a..f190cafce 100644 --- a/debian/copyright +++ b/debian/copyright @@ -177,6 +177,10 @@ Copyright: 2006-2010, QLogic Corp. 2013, Intel Corporation License: BSD-MIT or GPL-2 +Files: providers/ionic/* +Copyright: 2018-2025, Advanced Micro Devices, Inc. +License: BSD-MIT or GPL-2 + Files: providers/irdma/* Copyright: 2015-2023, Intel Corporation. License: BSD-MIT or GPL-2 diff --git a/debian/ibverbs-providers.install b/debian/ibverbs-providers.install index 42e939aed..38d870e9e 100644 --- a/debian/ibverbs-providers.install +++ b/debian/ibverbs-providers.install @@ -2,6 +2,7 @@ etc/libibverbs.d/ usr/lib/*/libefa.so.* usr/lib/*/libhns.so.* usr/lib/*/libibverbs/lib*-rdmav*.so +usr/lib/*/libionic.so.* usr/lib/*/libmana.so.* usr/lib/*/libmlx4.so.* usr/lib/*/libmlx5.so.* diff --git a/debian/ibverbs-providers.symbols b/debian/ibverbs-providers.symbols index 419bb9442..256d80fe2 100644 --- a/debian/ibverbs-providers.symbols +++ b/debian/ibverbs-providers.symbols @@ -191,3 +191,12 @@ libmana.so.1 ibverbs-providers #MINVER# MANA_1.0@MANA_1.0 41 manadv_init_obj@MANA_1.0 41 manadv_set_context_attr@MANA_1.0 41 +libionic.so.1 ibverbs-providers #MINVER# +* Build-Depends-Package: libibverbs-dev + IONIC_1.0@IONIC_1.0 59 + ionic_dv_ctx_get_udma_count@IONIC_1.0 59 + ionic_dv_ctx_get_udma_mask@IONIC_1.0 59 + ionic_dv_pd_get_udma_mask@IONIC_1.0 59 + ionic_dv_pd_set_udma_mask@IONIC_1.0 59 + ionic_dv_pd_set_sqcmb@IONIC_1.0 59 + ionic_dv_pd_set_rqcmb@IONIC_1.0 59 diff --git a/debian/libibverbs-dev.install b/debian/libibverbs-dev.install index e2009f5d0..0f29bcd07 100644 --- a/debian/libibverbs-dev.install +++ b/debian/libibverbs-dev.install @@ -2,6 +2,7 @@ usr/include/infiniband/arch.h usr/include/infiniband/efadv.h usr/include/infiniband/hnsdv.h usr/include/infiniband/ib_user_ioctl_verbs.h +usr/include/infiniband/ionic_dv.h usr/include/infiniband/manadv.h usr/include/infiniband/mlx4dv.h usr/include/infiniband/mlx5_api.h @@ -20,6 +21,8 @@ usr/lib/*/libhns.a usr/lib/*/libhns.so usr/lib/*/libibverbs*.so usr/lib/*/libibverbs.a +usr/lib/*/libionic.a +usr/lib/*/libionic.so* usr/lib/*/libmana.a usr/lib/*/libmana.so usr/lib/*/libmlx4.a @@ -29,6 +32,7 @@ usr/lib/*/libmlx5.so usr/lib/*/pkgconfig/libefa.pc usr/lib/*/pkgconfig/libhns.pc usr/lib/*/pkgconfig/libibverbs.pc +usr/lib/*/pkgconfig/libionic.pc usr/lib/*/pkgconfig/libmana.pc usr/lib/*/pkgconfig/libmlx4.pc usr/lib/*/pkgconfig/libmlx5.pc diff --git a/kernel-headers/CMakeLists.txt b/kernel-headers/CMakeLists.txt index 82c191cad..cd9c08cad 100644 --- a/kernel-headers/CMakeLists.txt +++ b/kernel-headers/CMakeLists.txt @@ -9,6 +9,7 @@ publish_internal_headers(rdma rdma/ib_user_mad.h rdma/ib_user_sa.h rdma/ib_user_verbs.h + rdma/ionic-abi.h rdma/irdma-abi.h rdma/mana-abi.h rdma/mlx4-abi.h @@ -70,6 +71,7 @@ rdma_kernel_provider_abi( rdma/erdma-abi.h rdma/hns-abi.h rdma/ib_user_verbs.h + rdma/ionic-abi.h rdma/irdma-abi.h rdma/mana-abi.h rdma/mlx4-abi.h diff --git a/kernel-headers/rdma/ib_user_ioctl_verbs.h b/kernel-headers/rdma/ib_user_ioctl_verbs.h index fe15bc7e9..89e6a3f13 100644 --- a/kernel-headers/rdma/ib_user_ioctl_verbs.h +++ b/kernel-headers/rdma/ib_user_ioctl_verbs.h @@ -255,6 +255,7 @@ enum rdma_driver_id { RDMA_DRIVER_SIW, RDMA_DRIVER_ERDMA, RDMA_DRIVER_MANA, + RDMA_DRIVER_IONIC, }; enum ib_uverbs_gid_type { diff --git a/kernel-headers/rdma/ionic-abi.h b/kernel-headers/rdma/ionic-abi.h new file mode 100644 index 000000000..7b589d3e9 --- /dev/null +++ b/kernel-headers/rdma/ionic-abi.h @@ -0,0 +1,115 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc */ + +#ifndef IONIC_ABI_H +#define IONIC_ABI_H + +#include + +#define IONIC_ABI_VERSION 1 + +#define IONIC_EXPDB_64 1 +#define IONIC_EXPDB_128 2 +#define IONIC_EXPDB_256 4 +#define IONIC_EXPDB_512 8 + +#define IONIC_EXPDB_SQ 1 +#define IONIC_EXPDB_RQ 2 + +#define IONIC_CMB_ENABLE 1 +#define IONIC_CMB_REQUIRE 2 +#define IONIC_CMB_EXPDB 4 +#define IONIC_CMB_WC 8 +#define IONIC_CMB_UC 16 + +struct ionic_ctx_req { + __u32 rsvd[2]; +}; + +struct ionic_ctx_resp { + __u32 rsvd; + __u32 page_shift; + + __aligned_u64 dbell_offset; + + __u16 version; + __u8 qp_opcodes; + __u8 admin_opcodes; + + __u8 sq_qtype; + __u8 rq_qtype; + __u8 cq_qtype; + __u8 admin_qtype; + + __u8 max_stride; + __u8 max_spec; + __u8 udma_count; + __u8 expdb_mask; + __u8 expdb_qtypes; + + __u8 rsvd2[3]; +}; + +struct ionic_qdesc { + __aligned_u64 addr; + __u32 size; + __u16 mask; + __u8 depth_log2; + __u8 stride_log2; +}; + +struct ionic_ah_resp { + __u32 ahid; + __u32 pad; +}; + +struct ionic_cq_req { + struct ionic_qdesc cq[2]; + __u8 udma_mask; + __u8 rsvd[7]; +}; + +struct ionic_cq_resp { + __u32 cqid[2]; + __u8 udma_mask; + __u8 rsvd[7]; +}; + +struct ionic_qp_req { + struct ionic_qdesc sq; + struct ionic_qdesc rq; + __u8 sq_spec; + __u8 rq_spec; + __u8 sq_cmb; + __u8 rq_cmb; + __u8 udma_mask; + __u8 rsvd[3]; +}; + +struct ionic_qp_resp { + __u32 qpid; + __u8 sq_cmb; + __u8 rq_cmb; + __u8 udma_idx; + __u8 rsvd[1]; + __aligned_u64 sq_cmb_offset; + __aligned_u64 rq_cmb_offset; +}; + +struct ionic_srq_req { + struct ionic_qdesc rq; + __u8 rq_spec; + __u8 rq_cmb; + __u8 udma_mask; + __u8 rsvd[5]; +}; + +struct ionic_srq_resp { + __u32 qpid; + __u8 rq_cmb; + __u8 udma_idx; + __u8 rsvd[2]; + __aligned_u64 rq_cmb_offset; +}; + +#endif /* IONIC_ABI_H */ diff --git a/libibverbs/verbs.h b/libibverbs/verbs.h index 821341242..b5681b53e 100644 --- a/libibverbs/verbs.h +++ b/libibverbs/verbs.h @@ -2298,7 +2298,7 @@ struct ibv_device **ibv_get_device_list(int *num_devices); */ #ifdef RDMA_STATIC_PROVIDERS #define _RDMA_STATIC_PREFIX_(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, \ - _12, _13, _14, _15, _16, _17, _18, _19, ...) \ + _12, _13, _14, _15, _16, _17, _18, _19, _20, ...) \ &verbs_provider_##_1, &verbs_provider_##_2, &verbs_provider_##_3, \ &verbs_provider_##_4, &verbs_provider_##_5, \ &verbs_provider_##_6, &verbs_provider_##_7, \ @@ -2307,11 +2307,12 @@ struct ibv_device **ibv_get_device_list(int *num_devices); &verbs_provider_##_12, &verbs_provider_##_13, \ &verbs_provider_##_14, &verbs_provider_##_15, \ &verbs_provider_##_16, &verbs_provider_##_17, \ - &verbs_provider_##_18, &verbs_provider_##_19 + &verbs_provider_##_18, &verbs_provider_##_19, \ + &verbs_provider_##_20 #define _RDMA_STATIC_PREFIX(arg) \ _RDMA_STATIC_PREFIX_(arg, none, none, none, none, none, none, none, \ none, none, none, none, none, none, none, none, \ - none, none, none) + none, none, none, none) struct verbs_devices_ops; extern const struct verbs_device_ops verbs_provider_bnxt_re; @@ -2331,6 +2332,7 @@ extern const struct verbs_device_ops verbs_provider_qedr; extern const struct verbs_device_ops verbs_provider_rxe; extern const struct verbs_device_ops verbs_provider_siw; extern const struct verbs_device_ops verbs_provider_vmw_pvrdma; +extern const struct verbs_device_ops verbs_provider_ionic; extern const struct verbs_device_ops verbs_provider_all; extern const struct verbs_device_ops verbs_provider_none; void ibv_static_providers(void *unused, ...); diff --git a/providers/ionic/CMakeLists.txt b/providers/ionic/CMakeLists.txt new file mode 100644 index 000000000..a28083fdc --- /dev/null +++ b/providers/ionic/CMakeLists.txt @@ -0,0 +1,14 @@ +rdma_shared_provider(ionic libionic.map + 1 1.0.${PACKAGE_VERSION} + ionic.c + ionic_verbs.c + ionic_memory.c + ionic_queue.c + ionic_dv.c +) + +publish_headers(infiniband + ionic_dv.h +) + +rdma_pkg_config("ionic" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}") diff --git a/providers/ionic/ionic-abi.h b/providers/ionic/ionic-abi.h new file mode 100644 index 000000000..e3c6933d4 --- /dev/null +++ b/providers/ionic/ionic-abi.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* + * Copyright (c) 2018-2025 Advanced Micro Devices, Inc. All rights reserved. + */ + +#ifndef __IONIC_ABI_H__ +#define __IONIC_ABI_H__ + +#include +#include +#include +#include + +#include "ionic_fw_types.h" + +DECLARE_DRV_CMD(uionic_ctx, IB_USER_VERBS_CMD_GET_CONTEXT, + ionic_ctx_req, ionic_ctx_resp); +DECLARE_DRV_CMD(uionic_ah, IB_USER_VERBS_CMD_CREATE_AH, + empty, ionic_ah_resp); +DECLARE_DRV_CMD(uionic_cq, IB_USER_VERBS_CMD_CREATE_CQ, + ionic_cq_req, ionic_cq_resp); +DECLARE_DRV_CMD(uionic_qp, IB_USER_VERBS_EX_CMD_CREATE_QP, + ionic_qp_req, ionic_qp_resp); + +#endif /* __IONIC_ABI_H__ */ diff --git a/providers/ionic/ionic.c b/providers/ionic/ionic.c new file mode 100644 index 000000000..a91fa19e4 --- /dev/null +++ b/providers/ionic/ionic.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* + * Copyright (c) 2018-2025 Advanced Micro Devices, Inc. All rights reserved. + */ + +#include +#include + +#include "ionic.h" + +static struct verbs_context *ionic_alloc_context(struct ibv_device *ibdev, + int cmd_fd, + void *private_data) +{ + struct ionic_ctx *ctx; + struct uionic_ctx req = {}; + struct uionic_ctx_resp resp = {}; + uint64_t mask; + int rc; + + ctx = verbs_init_and_alloc_context(ibdev, cmd_fd, ctx, vctx, + RDMA_DRIVER_IONIC); + if (!ctx) { + rc = errno; + goto err_ctx; + } + + rc = ibv_cmd_get_context(&ctx->vctx, &req.ibv_cmd, sizeof(req), + NULL, &resp.ibv_resp, sizeof(resp)); + if (rc) + goto err_cmd; + + ctx->pg_shift = resp.page_shift; + + if (resp.version < IONIC_MIN_RDMA_VERSION) { + verbs_err(&ctx->vctx, "ionic: Firmware RDMA Version %u\n", + resp.version); + verbs_err(&ctx->vctx, "ionic: Driver Min RDMA Version %u\n", + IONIC_MIN_RDMA_VERSION); + rc = EINVAL; + goto err_cmd; + } + + if (resp.version > IONIC_MAX_RDMA_VERSION) { + verbs_err(&ctx->vctx, "ionic: Firmware RDMA Version %u\n", + resp.version); + verbs_err(&ctx->vctx, "ionic: Driver Max RDMA Version %u\n", + IONIC_MAX_RDMA_VERSION); + rc = EINVAL; + goto err_cmd; + } + + ctx->version = resp.version; + ctx->opcodes = resp.qp_opcodes; + if (ctx->version == 1 && ctx->opcodes <= IONIC_V1_OP_BIND_MW) { + verbs_err(&ctx->vctx, "ionic: qp opcodes %d want min %d\n", + ctx->opcodes, IONIC_V1_OP_BIND_MW + 1); + rc = EINVAL; + goto err_cmd; + } + + if (resp.udma_count != 1 && resp.udma_count != 2) { + verbs_err(&ctx->vctx, "ionic: udma_count %d invalid\n", + resp.udma_count); + rc = EINVAL; + goto err_cmd; + } + ctx->udma_count = resp.udma_count; + + ctx->sq_qtype = resp.sq_qtype; + ctx->rq_qtype = resp.rq_qtype; + ctx->cq_qtype = resp.cq_qtype; + + ctx->max_stride = resp.max_stride; + + ctx->expdb_mask = resp.expdb_mask; + ctx->sq_expdb = !!(resp.expdb_qtypes & IONIC_EXPDB_SQ); + ctx->rq_expdb = !!(resp.expdb_qtypes & IONIC_EXPDB_RQ); + + mask = (1u << ctx->pg_shift) - 1; + ctx->dbpage_page = ionic_map_device(1u << ctx->pg_shift, cmd_fd, + resp.dbell_offset & ~mask); + if (!ctx->dbpage_page) { + rc = errno; + goto err_cmd; + } + ctx->dbpage = ctx->dbpage_page + (resp.dbell_offset & mask); + + pthread_mutex_init(&ctx->mut, NULL); + ionic_tbl_init(&ctx->qp_tbl); + + ionic_verbs_set_ops(ctx); + + ctx->spec = resp.max_spec; + if (ctx->spec < 0 || ctx->spec > 16) + ctx->spec = 0; + + verbs_debug(&ctx->vctx, "Attached to ctx %p", ctx); + return &ctx->vctx; + +err_cmd: + verbs_uninit_context(&ctx->vctx); +err_ctx: + errno = rc; + return NULL; +} + +static const struct verbs_match_ent cna_table[] = { + VERBS_DRIVER_ID(RDMA_DRIVER_IONIC), + {} +}; + +static struct verbs_device *ionic_alloc_device(struct verbs_sysfs_dev *sdev) +{ + struct ionic_dev *dev; + + static_assert(sizeof(struct ionic_v1_cqe) == 32, "bad size"); + static_assert(sizeof(struct ionic_v1_base_hdr) == 16, "bad size"); + static_assert(sizeof(struct ionic_v1_recv_bdy) == 48, "bad size"); + static_assert(sizeof(struct ionic_v1_common_bdy) == 48, "bad size"); + static_assert(sizeof(struct ionic_v1_atomic_bdy) == 48, "bad size"); + static_assert(sizeof(struct ionic_v1_bind_mw_bdy) == 48, "bad size"); + static_assert(sizeof(struct ionic_v1_wqe) == 64, "bad size"); + + dev = calloc(1, sizeof(*dev)); + if (!dev) + return NULL; + + dev->abi_ver = sdev->abi_ver; + + return &dev->vdev; +} + +static void ionic_uninit_device(struct verbs_device *vdev) +{ + struct ionic_dev *dev = to_ionic_dev(&vdev->device); + + free(dev); +} + +static const struct verbs_device_ops ionic_dev_ops = { + .name = "ionic", + .match_min_abi_version = IONIC_ABI_VERSION, + .match_max_abi_version = IONIC_ABI_VERSION, + .match_table = cna_table, + .alloc_device = ionic_alloc_device, + .uninit_device = ionic_uninit_device, + .alloc_context = ionic_alloc_context, +}; + +PROVIDER_DRIVER(ionic, ionic_dev_ops); diff --git a/providers/ionic/ionic.h b/providers/ionic/ionic.h new file mode 100644 index 000000000..d618645df --- /dev/null +++ b/providers/ionic/ionic.h @@ -0,0 +1,310 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* + * Copyright (c) 2018-2025 Advanced Micro Devices, Inc. All rights reserved. + */ + +#ifndef IONIC_H +#define IONIC_H + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "ionic-abi.h" + +#include "ionic_memory.h" +#include "ionic_queue.h" +#include "ionic_table.h" + +#include +#include +#include +#include + +#define IONIC_MIN_RDMA_VERSION 1 +#define IONIC_MAX_RDMA_VERSION 2 + +#define IONIC_META_LAST ((void *)1ul) +#define IONIC_META_POSTED ((void *)2ul) + +#define IONIC_CQ_GRACE 100 +#define IONIC_PAGE_SIZE 4096 + +#define IONIC_QUEUE_DEPTH_MAX 0xFFFF +#define IONIC_QUEUE_STRIDE_MAX 0x10000 + +/** IONIC_PD_TAG - tag used for parent domain resource allocation. */ +#define IONIC_PD_TAG ((uint64_t)RDMA_DRIVER_IONIC << 32) +#define IONIC_PD_TAG_CQ (IONIC_PD_TAG | 1) +#define IONIC_PD_TAG_SQ (IONIC_PD_TAG | 2) +#define IONIC_PD_TAG_RQ (IONIC_PD_TAG | 3) + +enum { + IONIC_CQ_SUPPORTED_WC_FLAGS = + IBV_WC_EX_WITH_BYTE_LEN | + IBV_WC_EX_WITH_IMM | + IBV_WC_EX_WITH_QP_NUM | + IBV_WC_EX_WITH_SRC_QP | + IBV_WC_EX_WITH_SLID | + IBV_WC_EX_WITH_SL | + IBV_WC_EX_WITH_DLID_PATH_BITS +}; + +struct ionic_ctx { + struct verbs_context vctx; + + int spec; + uint32_t pg_shift; + + int version; + uint8_t opcodes; + + uint8_t sq_qtype; + uint8_t rq_qtype; + uint8_t cq_qtype; + + uint8_t max_stride; + + uint8_t udma_count; + uint8_t expdb_mask; + bool sq_expdb; + bool rq_expdb; + + void *dbpage_page; + uint64_t *dbpage; + + pthread_mutex_t mut; + struct ionic_tbl_root qp_tbl; + + FILE *dbg_file; +}; + +struct ionic_pd { + struct ibv_pd ibpd; + struct ibv_pd *root_ibpd; + + uint8_t udma_mask; + uint8_t sq_cmb; + uint8_t rq_cmb; + + void *(*alloc)(struct ibv_pd *pd, void *pd_context, size_t size, + size_t alignment, uint64_t resource_type); + void (*free)(struct ibv_pd *pd, void *pd_context, void *ptr, + uint64_t resource_type); + void *pd_context; +}; + +struct ionic_cq { + struct ionic_vcq *vcq; + + uint32_t cqid; + + pthread_spinlock_t lock; + unsigned long cqseq; + + struct list_head poll_sq; + struct list_head poll_rq; + bool flush; + struct list_head flush_sq; + struct list_head flush_rq; + struct ionic_queue q; + bool color; + bool deferred_arm; + bool deferred_arm_sol_only; + bool lockfree; + int reserve; + int reserve_pending; + uint16_t arm_any_prod; + uint16_t arm_sol_prod; +}; + +struct ionic_vcq { + struct verbs_cq vcq; + struct ionic_cq cq[2]; + uint8_t udma_mask; + uint8_t poll_idx; + struct ibv_wc cur_wc; /* for use with start_poll/next_poll */ +}; + +struct ionic_sq_meta { + uint64_t wrid; + uint32_t len; + uint16_t seq; + uint8_t ibop; + uint8_t ibsts; + bool remote; + bool signal; + bool local_comp; +}; + +struct ionic_rq_meta { + struct ionic_rq_meta *next; + uint64_t wrid; +}; + +struct ionic_rq { + pthread_spinlock_t lock; + struct ionic_queue queue; + + void *cmb_ptr; + struct ionic_rq_meta *meta; + struct ionic_rq_meta *meta_head; + uint16_t *meta_idx; + + int spec; + uint16_t old_prod; + uint16_t cmb_prod; + uint8_t cmb; + bool flush; +}; + +struct ionic_sq { + pthread_spinlock_t lock; + struct ionic_queue queue; + + void *cmb_ptr; + struct ionic_sq_meta *meta; + uint16_t *msn_idx; + + int spec; + uint16_t old_prod; + uint16_t msn_prod; + uint16_t msn_cons; + uint16_t cmb_prod; + uint8_t cmb; + + bool flush; + bool flush_rcvd; + bool color; +}; + +struct ionic_qp { + struct verbs_qp vqp; + + uint32_t qpid; + uint8_t udma_idx; + bool has_sq; + bool has_rq; + bool lockfree; + + struct list_node cq_poll_sq; + struct list_node cq_poll_rq; + struct list_node cq_flush_sq; + struct list_node cq_flush_rq; + + struct ionic_sq sq; + struct ionic_rq rq; +}; + +struct ionic_ah { + struct ibv_ah ibah; + uint32_t ahid; +}; + +struct ionic_dev { + struct verbs_device vdev; + int abi_ver; +}; + +bool is_ionic_ctx(struct ibv_context *ibctx); + +static inline bool is_ionic_pd(struct ibv_pd *ibpd) +{ + return is_ionic_ctx(ibpd->context); +} + +static inline bool is_ionic_cq(struct ibv_cq *ibcq) +{ + return is_ionic_ctx(ibcq->context); +} + +static inline bool is_ionic_qp(struct ibv_qp *ibqp) +{ + return is_ionic_ctx(ibqp->context); +} + +static inline struct ionic_dev *to_ionic_dev(struct ibv_device *ibdev) +{ + return container_of(ibdev, struct ionic_dev, vdev.device); +} + +static inline struct ionic_ctx *to_ionic_ctx(struct ibv_context *ibctx) +{ + return container_of(ibctx, struct ionic_ctx, vctx.context); +} + +static inline struct ionic_pd *to_ionic_pd(struct ibv_pd *ibpd) +{ + return container_of(ibpd, struct ionic_pd, ibpd); +} + +static inline struct ibv_pd *ionic_root_ibpd(struct ionic_pd *pd) +{ + return pd->root_ibpd; +} + +static inline struct ibv_pd *to_ionic_root_ibpd(struct ibv_pd *ibpd) +{ + return ionic_root_ibpd(to_ionic_pd(ibpd)); +} + +static inline struct ionic_vcq *to_ionic_vcq_ex(struct ibv_cq_ex *ibcq) +{ + return container_of(ibcq, struct ionic_vcq, vcq.cq_ex); +} + +static inline struct ionic_vcq *to_ionic_vcq(struct ibv_cq *ibcq) +{ + return container_of(ibcq, struct ionic_vcq, vcq.cq); +} + +static inline struct ionic_cq *to_ionic_vcq_cq(struct ibv_cq *ibcq, + uint8_t udma_idx) +{ + return &to_ionic_vcq(ibcq)->cq[udma_idx]; +} + +static inline struct ionic_qp *to_ionic_qp(struct ibv_qp *ibqp) +{ + return container_of(ibqp, struct ionic_qp, vqp.qp); +} + +static inline struct ionic_ah *to_ionic_ah(struct ibv_ah *ibah) +{ + return container_of(ibah, struct ionic_ah, ibah); +} + +static inline bool ionic_ibop_is_local(enum ibv_wr_opcode op) +{ + return op == IBV_WR_LOCAL_INV || op == IBV_WR_BIND_MW; +} + +static inline uint8_t ionic_ctx_udma_mask(struct ionic_ctx *ctx) +{ + return BIT(ctx->udma_count) - 1; +} + +static inline void ionic_dbg_xdump(struct ionic_ctx *ctx, const char *str, + const void *ptr, size_t size) +{ + const uint8_t *ptr8 = ptr; + int i; + + for (i = 0; i < size; i += 8) + verbs_debug(&ctx->vctx, + "%s: %02x %02x %02x %02x %02x %02x %02x %02x", str, + ptr8[i + 0], ptr8[i + 1], ptr8[i + 2], ptr8[i + 3], + ptr8[i + 4], ptr8[i + 5], ptr8[i + 6], ptr8[i + 7]); +} + +/* ionic_verbs.h */ +void ionic_verbs_set_ops(struct ionic_ctx *ctx); + +#endif /* IONIC_H */ diff --git a/providers/ionic/ionic_dv.c b/providers/ionic/ionic_dv.c new file mode 100644 index 000000000..738fcc3e2 --- /dev/null +++ b/providers/ionic/ionic_dv.c @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* + * Copyright (c) 2018-2025 Advanced Micro Devices, Inc. All rights reserved. + */ + +#include "ionic.h" +#include "ionic_dv.h" + +uint8_t ionic_dv_ctx_get_udma_count(struct ibv_context *ibctx) +{ + if (!is_ionic_ctx(ibctx)) + return 0; + + return to_ionic_ctx(ibctx)->udma_count; +} + +uint8_t ionic_dv_ctx_get_udma_mask(struct ibv_context *ibctx) +{ + if (!is_ionic_ctx(ibctx)) + return 0; + + return ionic_ctx_udma_mask(to_ionic_ctx(ibctx)); +} + +uint8_t ionic_dv_pd_get_udma_mask(struct ibv_pd *ibpd) +{ + if (!is_ionic_pd(ibpd)) + return 0; + + return to_ionic_pd(ibpd)->udma_mask; +} + +int ionic_dv_pd_set_udma_mask(struct ibv_pd *ibpd, uint8_t udma_mask) +{ + if (!is_ionic_pd(ibpd)) + return EPERM; + + if (udma_mask & ~ionic_ctx_udma_mask(to_ionic_ctx(ibpd->context))) + return EINVAL; + + to_ionic_pd(ibpd)->udma_mask = udma_mask; + + return 0; +} + +static uint8_t ionic_dv_cmb_val(bool enable, bool expdb, bool require) +{ + uint8_t cmb = 0; + + if (enable) { + cmb = IONIC_CMB_ENABLE; + + if (expdb) + cmb |= IONIC_CMB_EXPDB; + + if (require) + cmb |= IONIC_CMB_REQUIRE; + } + + return cmb; +} + +int ionic_dv_pd_set_sqcmb(struct ibv_pd *ibpd, bool enable, bool expdb, bool require) +{ + struct ionic_ctx *ctx; + struct ionic_pd *pd; + + if (!is_ionic_pd(ibpd)) + return EPERM; + + ctx = to_ionic_ctx(ibpd->context); + pd = to_ionic_pd(ibpd); + + if (enable && expdb) { + if (require && !ctx->sq_expdb) + return EINVAL; + + expdb = ctx->sq_expdb; + } + + pd->sq_cmb = ionic_dv_cmb_val(enable, expdb, require); + + return 0; +} + +int ionic_dv_pd_set_rqcmb(struct ibv_pd *ibpd, bool enable, bool expdb, bool require) +{ + struct ionic_ctx *ctx; + struct ionic_pd *pd; + + if (!is_ionic_pd(ibpd)) + return EPERM; + + ctx = to_ionic_ctx(ibpd->context); + pd = to_ionic_pd(ibpd); + + if (enable && expdb) { + if (require && !ctx->rq_expdb) + return EINVAL; + + expdb = ctx->rq_expdb; + } + + pd->rq_cmb = ionic_dv_cmb_val(enable, expdb, require); + + return 0; +} diff --git a/providers/ionic/ionic_dv.h b/providers/ionic/ionic_dv.h new file mode 100644 index 000000000..a11f5a212 --- /dev/null +++ b/providers/ionic/ionic_dv.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* + * Copyright (c) 2018-2025 Advanced Micro Devices, Inc. All rights reserved. + */ + +#ifndef IONIC_DV_H +#define IONIC_DV_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * ionic_dv_ctx_get_udma_count - Get number of udma pipelines. + */ +uint8_t ionic_dv_ctx_get_udma_count(struct ibv_context *ibctx); + +/** + * ionic_dv_ctx_get_udma_mask - Get mask of udma pipeline ids. + */ +uint8_t ionic_dv_ctx_get_udma_mask(struct ibv_context *ibctx); + +/** + * ionic_dv_pd_get_udma_mask - Get mask of udma pipeline ids of pd or parent domain. + */ +uint8_t ionic_dv_pd_get_udma_mask(struct ibv_pd *ibpd); + +/** + * ionic_dv_pd_set_udma_mask - Restrict pipeline ids of pd or parent domain. + * + * Queues associated with this pd will be restricted to one of the pipelines enabled by + * the mask at the time of queue creation. + * + * Recommended usage is to create a pd, then parent domains of that pd for each different + * udma mask. Set the desired udma mask on each parent domain. Then, create queues + * associated with the parent domain with the desired udma mask. + * + * Alternative usage is to create a pd, and set the desired udma mask prior to creating + * each queue. Changing the udma mask of the pd has no effect on previously created + * queues. + */ +int ionic_dv_pd_set_udma_mask(struct ibv_pd *ibpd, uint8_t udma_mask); + +/** + * ionic_dv_pd_set_sqcmb - Specify send queue preference for controller memory bar. + * + * Send queues associated with this pd will use the controller memory bar according to + * this preference at the time of queue creation. + * + * @enable - Allow the use of the controller memory bar. + * @expdb - Allow the use of express doorbell optimizations. + * @require - Require preferences to be met, no fallback. + */ +int ionic_dv_pd_set_sqcmb(struct ibv_pd *ibpd, bool enable, bool expdb, bool require); + +/** + * ionic_dv_pd_set_rqcmb - Specify receive queue preference for controller memory bar. + * + * Receive queues associated with this pd will use the controller memory bar according to + * this preference at the time of queue creation. + * + * @enable - Allow the use of the controller memory bar. + * @expdb - Allow the use of express doorbell optimizations. + * @require - Require preferences to be met, no fallback. + */ +int ionic_dv_pd_set_rqcmb(struct ibv_pd *ibpd, bool enable, bool expdb, bool require); + +#ifdef __cplusplus +} +#endif + +#endif /* IONIC_DV_H */ diff --git a/providers/ionic/ionic_fw.h b/providers/ionic/ionic_fw.h new file mode 100644 index 000000000..6598213fc --- /dev/null +++ b/providers/ionic/ionic_fw.h @@ -0,0 +1,220 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* + * Copyright (c) 2018-2025 Advanced Micro Devices, Inc. All rights reserved. + */ + +#ifndef IONIC_FW_H +#define IONIC_FW_H + +#include "ionic_fw_types.h" + +static inline int to_ionic_mr_flags(int access) +{ + int flags = 0; + + if (access & IBV_ACCESS_LOCAL_WRITE) + flags |= IONIC_MRF_LOCAL_WRITE; + + if (access & IBV_ACCESS_REMOTE_READ) + flags |= IONIC_MRF_REMOTE_READ; + + if (access & IBV_ACCESS_REMOTE_WRITE) + flags |= IONIC_MRF_REMOTE_WRITE; + + if (access & IBV_ACCESS_REMOTE_ATOMIC) + flags |= IONIC_MRF_REMOTE_ATOMIC; + + if (access & IBV_ACCESS_MW_BIND) + flags |= IONIC_MRF_MW_BIND; + + if (access & IBV_ACCESS_ZERO_BASED) + flags |= IONIC_MRF_ZERO_BASED; + + return flags; +} + +static inline int ionic_to_ibv_status(int sts) +{ + switch (sts) { + case IONIC_STS_OK: + return IBV_WC_SUCCESS; + case IONIC_STS_LOCAL_LEN_ERR: + return IBV_WC_LOC_LEN_ERR; + case IONIC_STS_LOCAL_QP_OPER_ERR: + return IBV_WC_LOC_QP_OP_ERR; + case IONIC_STS_LOCAL_PROT_ERR: + return IBV_WC_LOC_PROT_ERR; + case IONIC_STS_WQE_FLUSHED_ERR: + return IBV_WC_WR_FLUSH_ERR; + case IONIC_STS_MEM_MGMT_OPER_ERR: + return IBV_WC_MW_BIND_ERR; + case IONIC_STS_BAD_RESP_ERR: + return IBV_WC_BAD_RESP_ERR; + case IONIC_STS_LOCAL_ACC_ERR: + return IBV_WC_LOC_ACCESS_ERR; + case IONIC_STS_REMOTE_INV_REQ_ERR: + return IBV_WC_REM_INV_REQ_ERR; + case IONIC_STS_REMOTE_ACC_ERR: + return IBV_WC_REM_ACCESS_ERR; + case IONIC_STS_REMOTE_OPER_ERR: + return IBV_WC_REM_OP_ERR; + case IONIC_STS_RETRY_EXCEEDED: + return IBV_WC_RETRY_EXC_ERR; + case IONIC_STS_RNR_RETRY_EXCEEDED: + return IBV_WC_RNR_RETRY_EXC_ERR; + case IONIC_STS_XRC_VIO_ERR: + default: + return IBV_WC_GENERAL_ERR; + } +} + +static inline bool ionic_v1_cqe_color(struct ionic_v1_cqe *cqe) +{ + return !!(cqe->qid_type_flags & htobe32(IONIC_V1_CQE_COLOR)); +} + +static inline bool ionic_v1_cqe_error(struct ionic_v1_cqe *cqe) +{ + return !!(cqe->qid_type_flags & htobe32(IONIC_V1_CQE_ERROR)); +} + +static inline bool ionic_v1_cqe_recv_is_ipv4(struct ionic_v1_cqe *cqe) +{ + return !!(cqe->recv.src_qpn_op & + htobe32(IONIC_V1_CQE_RECV_IS_IPV4)); +} + +static inline bool ionic_v1_cqe_recv_is_vlan(struct ionic_v1_cqe *cqe) +{ + return !!(cqe->recv.src_qpn_op & + htobe32(IONIC_V1_CQE_RECV_IS_VLAN)); +} + +static inline void ionic_v1_cqe_clean(struct ionic_v1_cqe *cqe) +{ + cqe->qid_type_flags |= htobe32(~0u << IONIC_V1_CQE_QID_SHIFT); +} + +static inline uint32_t ionic_v1_cqe_qtf(struct ionic_v1_cqe *cqe) +{ + return be32toh(cqe->qid_type_flags); +} + +static inline uint8_t ionic_v1_cqe_qtf_type(uint32_t qtf) +{ + return (qtf >> IONIC_V1_CQE_TYPE_SHIFT) & IONIC_V1_CQE_TYPE_MASK; +} + +static inline uint32_t ionic_v1_cqe_qtf_qid(uint32_t qtf) +{ + return qtf >> IONIC_V1_CQE_QID_SHIFT; +} + +static inline size_t ionic_v1_send_wqe_min_size(int min_sge, int min_data, + int spec, bool expdb) +{ + size_t sz_wqe, sz_sgl, sz_data; + + if (spec > IONIC_V1_SPEC_FIRST_SGE) + min_sge += IONIC_V1_SPEC_FIRST_SGE; + + if (expdb) { + min_sge += 1; + min_data += IONIC_EXP_DBELL_SZ; + } + + sz_wqe = sizeof(struct ionic_v1_wqe); + sz_sgl = offsetof(struct ionic_v1_wqe, common.pld.sgl[min_sge]); + sz_data = offsetof(struct ionic_v1_wqe, common.pld.data[min_data]); + + if (sz_sgl > sz_wqe) + sz_wqe = sz_sgl; + + if (sz_data > sz_wqe) + sz_wqe = sz_data; + + return roundup_pow_of_two(sz_wqe); +} + +static inline int ionic_v1_send_wqe_max_sge(uint8_t stride_log2, int spec, bool expdb) +{ + struct ionic_v1_wqe *wqe = (void *)0; + struct ionic_sge *sge = (void *)(uintptr_t)(1ull << stride_log2); + int num_sge = 0; + + if (expdb) + sge -= 1; + + if (spec > IONIC_V1_SPEC_FIRST_SGE) + num_sge = IONIC_V1_SPEC_FIRST_SGE; + + num_sge = sge - &wqe->common.pld.sgl[num_sge]; + + if (spec && num_sge > spec) + num_sge = spec; + + return num_sge; +} + +static inline int ionic_v1_send_wqe_max_data(uint8_t stride_log2, bool expdb) +{ + struct ionic_v1_wqe *wqe = (void *)0; + __u8 *data = (void *)(uintptr_t)(1ull << stride_log2); + + if (expdb) + data -= IONIC_EXP_DBELL_SZ; + + return data - wqe->common.pld.data; +} + +static inline size_t ionic_v1_recv_wqe_min_size(int min_sge, int spec, bool expdb) +{ + size_t sz_wqe, sz_sgl; + + if (spec > IONIC_V1_SPEC_FIRST_SGE) + min_sge += IONIC_V1_SPEC_FIRST_SGE; + + if (expdb) + min_sge += 1; + + sz_wqe = sizeof(struct ionic_v1_wqe); + sz_sgl = offsetof(struct ionic_v1_wqe, recv.pld.sgl[min_sge]); + + if (sz_sgl > sz_wqe) + sz_wqe = sz_sgl; + + return sz_wqe; +} + +static inline int ionic_v1_recv_wqe_max_sge(uint8_t stride_log2, int spec, bool expdb) +{ + struct ionic_v1_wqe *wqe = (void *)0; + struct ionic_sge *sge = (void *)(uintptr_t)(1ull << stride_log2); + int num_sge = 0; + + if (expdb) + sge -= 1; + + if (spec > IONIC_V1_SPEC_FIRST_SGE) + num_sge = IONIC_V1_SPEC_FIRST_SGE; + + num_sge = sge - &wqe->recv.pld.sgl[num_sge]; + + if (spec && num_sge > spec) + num_sge = spec; + + return num_sge; +} + +static inline int ionic_v1_use_spec_sge(int min_sge, int spec) +{ + if (!spec || min_sge > spec) + return 0; + + if (min_sge <= IONIC_V1_SPEC_FIRST_SGE) + return IONIC_V1_SPEC_FIRST_SGE; + + return spec; +} + +#endif /* IONIC_FW_H */ diff --git a/providers/ionic/ionic_fw_types.h b/providers/ionic/ionic_fw_types.h new file mode 100644 index 000000000..f715d8583 --- /dev/null +++ b/providers/ionic/ionic_fw_types.h @@ -0,0 +1,256 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* + * Copyright (c) 2018-2025 Advanced Micro Devices, Inc. All rights reserved. + */ + +#ifndef IONIC_FW_TYPES_H +#define IONIC_FW_TYPES_H + +#define IONIC_EXP_DBELL_SZ 8 + +/* common to all versions */ + +/* wqe scatter gather element */ +struct ionic_sge { + __be64 va; + __be32 len; + __be32 lkey; +}; + +/* admin queue mr type */ +enum ionic_mr_flags { + /* bits that determine mr access */ + IONIC_MRF_LOCAL_WRITE = 1u << 0, + IONIC_MRF_REMOTE_WRITE = 1u << 1, + IONIC_MRF_REMOTE_READ = 1u << 2, + IONIC_MRF_REMOTE_ATOMIC = 1u << 3, + IONIC_MRF_MW_BIND = 1u << 4, + IONIC_MRF_ZERO_BASED = 1u << 5, + IONIC_MRF_ON_DEMAND = 1u << 6, + IONIC_MRF_PB = 1u << 7, + IONIC_MRF_ACCESS_MASK = (1u << 12) - 1, + + /* bits that determine mr type */ + IONIC_MRF_IS_MW = 1u << 14, + IONIC_MRF_INV_EN = 1u << 15, + + /* base flags combinations for mr types */ + IONIC_MRF_USER_MR = 0, + IONIC_MRF_PHYS_MR = IONIC_MRF_INV_EN, + IONIC_MRF_MW_1 = IONIC_MRF_IS_MW, + IONIC_MRF_MW_2 = IONIC_MRF_IS_MW | IONIC_MRF_INV_EN, +}; + +/* cqe status indicated in status_length field when err bit is set */ +enum ionic_status { + IONIC_STS_OK, + IONIC_STS_LOCAL_LEN_ERR, + IONIC_STS_LOCAL_QP_OPER_ERR, + IONIC_STS_LOCAL_PROT_ERR, + IONIC_STS_WQE_FLUSHED_ERR, + IONIC_STS_MEM_MGMT_OPER_ERR, + IONIC_STS_BAD_RESP_ERR, + IONIC_STS_LOCAL_ACC_ERR, + IONIC_STS_REMOTE_INV_REQ_ERR, + IONIC_STS_REMOTE_ACC_ERR, + IONIC_STS_REMOTE_OPER_ERR, + IONIC_STS_RETRY_EXCEEDED, + IONIC_STS_RNR_RETRY_EXCEEDED, + IONIC_STS_XRC_VIO_ERR, +}; + + +/* fw abi v1 */ + +/* data payload part of v1 wqe */ +union ionic_v1_pld { + struct ionic_sge sgl[2]; + __be32 spec32[8]; + __be16 spec16[16]; + __u8 data[32]; +}; + +/* completion queue v1 cqe */ +struct ionic_v1_cqe { + union { + struct { + __u64 wqe_id; + __be32 src_qpn_op; + __u8 src_mac[6]; + __be16 vlan_tag; + __be32 imm_data_rkey; + } recv; + struct { + __u8 rsvd[4]; + __be32 msg_msn; + __u8 rsvd2[8]; + __u64 npg_wqe_id; + } send; + }; + __be32 status_length; + __be32 qid_type_flags; +}; + +/* bits for cqe recv */ +enum ionic_v1_cqe_src_qpn_bits { + IONIC_V1_CQE_RECV_QPN_MASK = 0xffffff, + IONIC_V1_CQE_RECV_OP_SHIFT = 24, + + /* MASK could be 0x3, but need 0x1f for makeshift values: + * OP_TYPE_RDMA_OPER_WITH_IMM, OP_TYPE_SEND_RCVD + */ + IONIC_V1_CQE_RECV_OP_MASK = 0x1f, + IONIC_V1_CQE_RECV_OP_SEND = 0, + IONIC_V1_CQE_RECV_OP_SEND_INV = 1, + IONIC_V1_CQE_RECV_OP_SEND_IMM = 2, + IONIC_V1_CQE_RECV_OP_RDMA_IMM = 3, + + IONIC_V1_CQE_RECV_IS_IPV4 = 1u << (7 + IONIC_V1_CQE_RECV_OP_SHIFT), + IONIC_V1_CQE_RECV_IS_VLAN = 1u << (6 + IONIC_V1_CQE_RECV_OP_SHIFT), +}; + +/* bits for cqe qid_type_flags */ +enum ionic_v1_cqe_qtf_bits { + IONIC_V1_CQE_COLOR = 1u << 0, + IONIC_V1_CQE_ERROR = 1u << 1, + IONIC_V1_CQE_TYPE_SHIFT = 5, + IONIC_V1_CQE_TYPE_MASK = 0x7, + IONIC_V1_CQE_QID_SHIFT = 8, + + IONIC_V1_CQE_TYPE_RECV = 1, + IONIC_V1_CQE_TYPE_SEND_MSN = 2, + IONIC_V1_CQE_TYPE_SEND_NPG = 3, + IONIC_V1_CQE_TYPE_RECV_INDIR = 4, +}; + +/* v1 base wqe header */ +struct ionic_v1_base_hdr { + __u64 wqe_id; + __u8 op; + __u8 num_sge_key; + __be16 flags; + __be32 imm_data_key; +}; + +/* v1 receive wqe body */ +struct ionic_v1_recv_bdy { + __u8 rsvd[16]; + union ionic_v1_pld pld; +}; + +/* v1 send/rdma wqe body (common, has sgl) */ +struct ionic_v1_common_bdy { + union { + struct { + __be32 ah_id; + __be32 dest_qpn; + __be32 dest_qkey; + } send; + struct { + __be32 remote_va_high; + __be32 remote_va_low; + __be32 remote_rkey; + } rdma; + }; + __be32 length; + union ionic_v1_pld pld; +}; + +/* v1 atomic wqe body */ +struct ionic_v1_atomic_bdy { + __be32 remote_va_high; + __be32 remote_va_low; + __be32 remote_rkey; + __be32 swap_add_high; + __be32 swap_add_low; + __be32 compare_high; + __be32 compare_low; + __u8 rsvd[4]; + struct ionic_sge sge; +}; + +/* v2 atomic wqe body */ +struct ionic_v2_atomic_bdy { + __be32 remote_va_high; + __be32 remote_va_low; + __be32 remote_rkey; + __be32 swap_add_high; + __be32 swap_add_low; + __be32 compare_high; + __be32 compare_low; + __be32 lkey; + __be64 local_va; + __u8 rsvd_expdb[8]; +}; + +/* v1 bind mw wqe body */ +struct ionic_v1_bind_mw_bdy { + __be64 va; + __be64 length; + __be32 lkey; + __be16 flags; + __u8 rsvd[26]; +}; + +/* v1 send/recv wqe */ +struct ionic_v1_wqe { + struct ionic_v1_base_hdr base; + union { + struct ionic_v1_recv_bdy recv; + struct ionic_v1_common_bdy common; + struct ionic_v1_atomic_bdy atomic; + struct ionic_v2_atomic_bdy atomic_v2; + struct ionic_v1_bind_mw_bdy bind_mw; + }; +}; + +/* queue pair v1 send opcodes */ +enum ionic_v1_op { + IONIC_V1_OP_SEND, + IONIC_V1_OP_SEND_INV, + IONIC_V1_OP_SEND_IMM, + IONIC_V1_OP_RDMA_READ, + IONIC_V1_OP_RDMA_WRITE, + IONIC_V1_OP_RDMA_WRITE_IMM, + IONIC_V1_OP_ATOMIC_CS, + IONIC_V1_OP_ATOMIC_FA, + IONIC_V1_OP_REG_MR, + IONIC_V1_OP_LOCAL_INV, + IONIC_V1_OP_BIND_MW, + + /* flags */ + IONIC_V1_FLAG_FENCE = 1u << 0, + IONIC_V1_FLAG_SOL = 1u << 1, + IONIC_V1_FLAG_INL = 1u << 2, + IONIC_V1_FLAG_SIG = 1u << 3, + IONIC_V1_FLAG_COLOR = 1u << 4, + + /* flags last four bits for sgl spec format */ + IONIC_V1_FLAG_SPEC32 = (1u << 12), + IONIC_V1_FLAG_SPEC16 = (2u << 12), + IONIC_V1_SPEC_FIRST_SGE = 2, +}; + +/* queue pair v2 send opcodes */ +enum ionic_v2_op { + IONIC_V2_OPSL_OUT = 0x20, + IONIC_V2_OPSL_IMM = 0x40, + IONIC_V2_OPSL_INV = 0x80, + + IONIC_V2_OP_SEND = 0x0 | IONIC_V2_OPSL_OUT, + IONIC_V2_OP_SEND_IMM = IONIC_V2_OP_SEND | IONIC_V2_OPSL_IMM, + IONIC_V2_OP_SEND_INV = IONIC_V2_OP_SEND | IONIC_V2_OPSL_INV, + + IONIC_V2_OP_RDMA_WRITE = 0x1 | IONIC_V2_OPSL_OUT, + IONIC_V2_OP_RDMA_WRITE_IMM = IONIC_V2_OP_RDMA_WRITE | IONIC_V2_OPSL_IMM, + + IONIC_V2_OP_RDMA_READ = 0x2, + + IONIC_V2_OP_ATOMIC_CS = 0x4, + IONIC_V2_OP_ATOMIC_FA = 0x5, + IONIC_V2_OP_REG_MR = 0x6, + IONIC_V2_OP_LOCAL_INV = 0x7, + IONIC_V2_OP_BIND_MW = 0x8, +}; + +#endif /* IONIC_FW_TYPES_H */ diff --git a/providers/ionic/ionic_memory.c b/providers/ionic/ionic_memory.c new file mode 100644 index 000000000..f84964c2f --- /dev/null +++ b/providers/ionic/ionic_memory.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* + * Copyright (c) 2018-2025 Advanced Micro Devices, Inc. All rights reserved. + */ + +#include +#include + +#include "ionic_memory.h" + +#define IONIC_ANON_MFLAGS (MAP_PRIVATE | MAP_ANONYMOUS) +#define IONIC_ANON_MPROT (PROT_READ | PROT_WRITE) + +#define IONIC_DEV_MFLAGS MAP_SHARED +#define IONIC_DEV_MPROT PROT_WRITE + +void *ionic_map_anon(size_t size) +{ + void *ptr; + int rc; + + ptr = mmap(NULL, size, IONIC_ANON_MPROT, IONIC_ANON_MFLAGS, -1, 0); + if (ptr == MAP_FAILED) + return NULL; + + rc = ibv_dontfork_range(ptr, size); + if (rc) { + munmap(ptr, size); + errno = rc; + return NULL; + } + + return ptr; +} + +void *ionic_map_device(size_t size, int fd, size_t offset) +{ + void *ptr; + int rc; + + ptr = mmap(NULL, size, IONIC_DEV_MPROT, IONIC_DEV_MFLAGS, fd, offset); + if (ptr == MAP_FAILED) + return NULL; + + rc = ibv_dontfork_range(ptr, size); + if (rc) { + munmap(ptr, size); + errno = rc; + return NULL; + } + + return ptr; +} + +void ionic_unmap(void *ptr, size_t size) +{ + if (ptr) { + ibv_dofork_range(ptr, size); + munmap(ptr, size); + } +} diff --git a/providers/ionic/ionic_memory.h b/providers/ionic/ionic_memory.h new file mode 100644 index 000000000..290af8e06 --- /dev/null +++ b/providers/ionic/ionic_memory.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* + * Copyright (c) 2018-2025 Advanced Micro Devices, Inc. All rights reserved. + */ + +#ifndef IONIC_MEMORY_H +#define IONIC_MEMORY_H + +#include + +/** + * ionic_map_anon() - Allocate page-aligned anonymous memory + * @size: Size to allocate + * + * Return: pointer to first page of memory, or NULL + */ +void *ionic_map_anon(size_t size); + +/** + * ionic_map_device() - Map device memory via fd and offset + * @size: Size to map + * @fd: File descriptor representing the device context + * @offset: Offset provided by the kernel as a handle for the mapping + * + * The mapping may be write-only, and may use write-combining. + * + * Return: pointer to first page of the mapping, or NULL + */ +void *ionic_map_device(size_t size, int fd, size_t offset); + +/** + * ionic_unmap() - Unmap anonymous or device memory + * @ptr: Pointer to the first page, or NULL (do nothing) + * @size: Size of the map + */ +void ionic_unmap(void *ptr, size_t size); + +#endif /* IONIC_MEMORY_H */ diff --git a/providers/ionic/ionic_queue.c b/providers/ionic/ionic_queue.c new file mode 100644 index 000000000..28458a618 --- /dev/null +++ b/providers/ionic/ionic_queue.c @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* + * Copyright (c) 2018-2025 Advanced Micro Devices, Inc. All rights reserved. + */ + +#include +#include + +#include "ionic.h" +#include "ionic_queue.h" +#include "ionic_memory.h" + +static void ionic_queue_map(struct ionic_queue *q, struct ionic_pd *pd, uint64_t pd_tag, int stride) +{ + if (pd && pd->alloc) { + size_t align = IONIC_PAGE_SIZE; + + if (align < stride) + align = stride; + + q->ptr = pd->alloc(&pd->ibpd, pd->pd_context, q->size, align, pd_tag); + if (q->ptr != IBV_ALLOCATOR_USE_DEFAULT) { + q->pd = pd; + q->pd_tag = pd_tag; + return; + } + } + + q->ptr = ionic_map_anon(q->size); +} + +static void ionic_queue_unmap(struct ionic_queue *q) +{ + struct ionic_pd *pd = q->pd; + + if (pd) { + pd->free(&pd->ibpd, pd->pd_context, q->ptr, q->pd_tag); + q->ptr = NULL; + q->pd = NULL; + q->pd_tag = 0; + return; + } + + ionic_unmap(q->ptr, q->size); +} + +int ionic_queue_init(struct ionic_queue *q, struct ionic_pd *pd, + uint64_t pd_tag, int pg_shift, int depth, size_t stride) +{ + if (depth < 0 || depth > IONIC_QUEUE_DEPTH_MAX) + return -EINVAL; + + if (stride == 0 || stride > IONIC_QUEUE_STRIDE_MAX) + return -EINVAL; + + if (depth == 0) + depth = 1; + + q->depth_log2 = ilog32(depth); + q->stride_log2 = ilog64(stride - 1); + + if (q->depth_log2 + q->stride_log2 < pg_shift) + q->depth_log2 = pg_shift - q->stride_log2; + + q->size = BIT_ULL(q->depth_log2 + q->stride_log2); + q->mask = BIT(q->depth_log2) - 1; + + ionic_queue_map(q, pd, pd_tag, stride); + if (!q->ptr) + return errno; + + q->prod = 0; + q->cons = 0; + q->dbell = 0; + + return 0; +} + +void ionic_queue_destroy(struct ionic_queue *q) +{ + ionic_queue_unmap(q); +} diff --git a/providers/ionic/ionic_queue.h b/providers/ionic/ionic_queue.h new file mode 100644 index 000000000..02ae808f7 --- /dev/null +++ b/providers/ionic/ionic_queue.h @@ -0,0 +1,237 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* + * Copyright (c) 2018-2025 Advanced Micro Devices, Inc. All rights reserved. + */ + +#ifndef IONIC_QUEUE_H +#define IONIC_QUEUE_H + +#include +#include +#include +#include +#include +#include + +#define IONIC_QID_MASK (BIT_ULL(24) - 1) +#define IONIC_DBELL_QID_SHIFT 24 +#define IONIC_DBELL_RING_ARM BIT_ULL(16) +#define IONIC_DBELL_RING_SONLY BIT_ULL(17) + +struct ionic_pd; + +/** + * struct ionic_queue - Ring buffer used between device and driver + * @ptr: Buffer virtual address + * @prod: Driver position in the queue + * @cons: Device position in the queue + * @mask: Capacity of the queue, subtracting the hole + * This value is equal to ((1 << depth_log2) - 1) + * @depth_log2: Log base two size depth of the queue + * @stride_log2: Log base two size of an element in the queue + * @dbell: Doorbell identifying bits + */ +struct ionic_queue { + size_t size; + void *ptr; + uint16_t prod; + uint16_t cons; + uint16_t mask; + uint8_t depth_log2; + uint8_t stride_log2; + uint64_t dbell; + + struct ionic_pd *pd; + uint64_t pd_tag; +}; + +/** + * ionic_queue_init() - Initialize user space queue + * @q: Uninitialized queue structure + * @pd: Parent domain for descriptor ring memory. + * @pd_tag: Identify the purpose of the ring memory. + * @pg_shift: Host page shift for buffer size-alignment and mapping + * @depth: Depth of the queue + * @stride: Size of each element of the queue + * + * Return: status code + */ +int ionic_queue_init(struct ionic_queue *q, struct ionic_pd *pd, + uint64_t pd_tag, int pg_shift, int depth, size_t stride); + +/** + * ionic_queue_destroy() - Destroy user space queue + * @q: Queue structure + * + * Return: status code + */ +void ionic_queue_destroy(struct ionic_queue *q); + +/** + * ionic_queue_empty() - Test if queue is empty + * @q: Queue structure + * + * This is only valid for to-device queues. + * + * Return: is empty + */ +static inline bool ionic_queue_empty(struct ionic_queue *q) +{ + return q->prod == q->cons; +} + +/** + * ionic_queue_length() - Get the current length of the queue + * @q: Queue structure + * + * This is only valid for to-device queues. + * + * Return: length + */ +static inline uint16_t ionic_queue_length(struct ionic_queue *q) +{ + return (q->prod - q->cons) & q->mask; +} + +/** + * ionic_queue_length_remaining() - Get the remaining length of the queue + * @q: Queue structure + * + * This is only valid for to-device queues. + * + * Return: length remaining + */ +static inline uint16_t ionic_queue_length_remaining(struct ionic_queue *q) +{ + return q->mask - ionic_queue_length(q); +} + +/** + * ionic_queue_full() - Test if queue is full + * @q: Queue structure + * + * This is only valid for to-device queues. + * + * Return: is full + */ +static inline bool ionic_queue_full(struct ionic_queue *q) +{ + return q->mask == ionic_queue_length(q); +} + +/** + * ionic_color_wrap() - Flip the color if prod is wrapped + * @prod: Queue index just after advancing + * @color: Queue color just prior to advancing the index + * + * Return: color after advancing the index + */ +static inline bool ionic_color_wrap(uint16_t prod, bool color) +{ + /* logical xor color with (prod == 0) */ + return color != (prod == 0); +} + +/** + * ionic_queue_at() - Get the element at the given index + * @q: Queue structure + * @idx: Index in the queue + * + * The index must be within the bounds of the queue. It is not checked here. + * + * Return: pointer to element at index + */ +static inline void *ionic_queue_at(struct ionic_queue *q, uint16_t idx) +{ + return q->ptr + ((unsigned long)idx << q->stride_log2); +} + +/** + * ionic_queue_at_prod() - Get the element at the producer index + * @q: Queue structure + * + * Return: pointer to element at producer index + */ +static inline void *ionic_queue_at_prod(struct ionic_queue *q) +{ + return ionic_queue_at(q, q->prod); +} + +/** + * ionic_queue_at_cons() - Get the element at the consumer index + * @q: Queue structure + * + * Return: pointer to element at consumer index + */ +static inline void *ionic_queue_at_cons(struct ionic_queue *q) +{ + return ionic_queue_at(q, q->cons); +} + +/** + * ionic_queue_next() - Compute the next index + * @q: Queue structure + * @idx: Index + * + * Return: next index after idx + */ +static inline uint16_t ionic_queue_next(struct ionic_queue *q, uint16_t idx) +{ + return (idx + 1) & q->mask; +} + +/** + * ionic_queue_produce() - Increase the producer index + * @q: Queue structure + * + * Caller must ensure that the queue is not full. It is not checked here. + */ +static inline void ionic_queue_produce(struct ionic_queue *q) +{ + q->prod = ionic_queue_next(q, q->prod); +} + +/** + * ionic_queue_consume() - Increase the consumer index + * @q: Queue structure + * + * Caller must ensure that the queue is not empty. It is not checked here. + * + * This is only valid for to-device queues. + */ +static inline void ionic_queue_consume(struct ionic_queue *q) +{ + q->cons = ionic_queue_next(q, q->cons); +} + +/** + * ionic_queue_dbell_init() - Initialize doorbell bits for queue id + * @q: Queue structure + * @qid: Queue identifying number + */ +static inline void ionic_queue_dbell_init(struct ionic_queue *q, + uint32_t qid) +{ + q->dbell = ((uint64_t)qid & IONIC_QID_MASK) << IONIC_DBELL_QID_SHIFT; +} + +/** + * ionic_queue_dbell_val() - Get current doorbell update value + * @q: Queue structure + */ +static inline uint64_t ionic_queue_dbell_val(struct ionic_queue *q) +{ + return q->dbell | q->prod; +} + +/** + * ionic_dbell_ring() - Write the doorbell value to register + * @dbreg: Doorbell register + * @val: Doorbell value from queue + */ +static inline void ionic_dbell_ring(uint64_t *dbreg, uint64_t val) +{ + mmio_write64_le(dbreg, htole64(val)); +} + +#endif /* IONIC_QUEUE_H */ diff --git a/providers/ionic/ionic_table.h b/providers/ionic/ionic_table.h new file mode 100644 index 000000000..354935b65 --- /dev/null +++ b/providers/ionic/ionic_table.h @@ -0,0 +1,244 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* + * Copyright (c) 2018-2025 Advanced Micro Devices, Inc. All rights reserved. + */ + +#ifndef IONIC_TABLE_H +#define IONIC_TABLE_H + +#include +#include +#include +#include + +/* Number of valid bits in a key */ +#define TBL_KEY_SHIFT 24 + +/* Number bits used for index in a node */ +#define TBL_NODE_SHIFT 12 + +#define TBL_NODE_MASK (BIT(TBL_NODE_SHIFT) - 1u) +#define TBL_NODE_CAPACITY BIT(TBL_NODE_SHIFT) +#define TBL_ROOT_CAPACITY BIT(TBL_KEY_SHIFT - TBL_NODE_SHIFT) + +struct ionic_tbl_node { + void *val[TBL_NODE_CAPACITY]; +}; + +struct ionic_tbl_root { + /* for lookup in table */ + struct ionic_tbl_node *node[TBL_ROOT_CAPACITY]; + + /* for insertion and deletion in table */ + int refcount[TBL_ROOT_CAPACITY]; + struct ionic_tbl_node *free_node; +}; + +/** + * ionic_tbl_init() - Initialize a table + * @tbl: Table root + */ +static inline void ionic_tbl_init(struct ionic_tbl_root *tbl) +{ + uint32_t node_i; + + tbl->free_node = NULL; + + for (node_i = 0; node_i < TBL_ROOT_CAPACITY; ++node_i) { + tbl->node[node_i] = NULL; + tbl->refcount[node_i] = 0; + } +} + +/** + * ionic_tbl_init() - Destroy the table, which should be empty + * @tbl: Table root + */ +static inline void ionic_tbl_destroy(struct ionic_tbl_root *tbl) +{ + uint32_t node_i; + + /* The table should be empty. If not empty, it means the context is + * being destroyed, but there are qps still in the table that have not + * been destroyed. + * + * The interface is such that freeing the context must succeed, so here + * will make a best effort to free table resources. Any qps that were + * not destroyed, however will still refer to the context after it is + * freed. Those qps must not be used, not even for ibv_destroy_qp, or + * the application will likely crash. + * + * This best-effort freeing of resources replaces an assert. The + * assert was seen in perftest, which will destroy the context even if + * there is an error destroying a qp or other resource. + */ + for (node_i = 0; node_i < TBL_ROOT_CAPACITY; ++node_i) { + if (!tbl->node[node_i]) + continue; + + free(tbl->node[node_i]); + } + + free(tbl->free_node); +} + +/** + * ionic_tbl_lookup() - Lookup value for key in the table + * @tbl: Table root + * @key: Key for lookup + * + * Synopsis: + * + * pthread_spin_lock(&my_table_lock); + * val = ionic_tbl_lookup(&my_table, key); + * if (val) + * my_val_routine(val); + * pthread_spin_unlock(&my_table_lock); + * + * Return: Value for key + */ +static inline void *ionic_tbl_lookup(struct ionic_tbl_root *tbl, uint32_t key) +{ + uint32_t node_i = key >> TBL_NODE_SHIFT; + + if (unlikely(key >> TBL_KEY_SHIFT)) + return NULL; + + if (unlikely(!tbl->node[node_i])) + return NULL; + + return tbl->node[node_i]->val[key & TBL_NODE_MASK]; +} + +/** + * ionic_tbl_alloc_node() - Allocate the free node prior to insertion + * @tbl: Table root + * + * This should be called before inserting. + * + * Synopsis: see ionic_tbl_insert() + */ +static inline void ionic_tbl_alloc_node(struct ionic_tbl_root *tbl) +{ + if (!tbl->free_node) + tbl->free_node = calloc(1, sizeof(*tbl->free_node)); +} + +/** + * ionic_tbl_free_node() - Free the free node prior to deletion + * @tbl: Table root + * + * This should be called before deleting. + * + * Synopsis: see ionic_tbl_delete() + */ +static inline void ionic_tbl_free_node(struct ionic_tbl_root *tbl) +{ + free(tbl->free_node); + tbl->free_node = NULL; +} + +/** + * ionic_tbl_insert() - Insert a value for key in the table + * @tbl: Table root + * @val: Value to insert + * @key: Key to insert + * + * The tbl->free_node must not be null when inserting. + * + * Synopsis: + * + * pthread_mutex_lock(&my_table_mut); + * ionic_tbl_alloc_node(&my_table); + * ionic_tbl_insert(&my_table, val, key); + * pthread_mutex_unlock(&my_table_mut); + * + * pthread_spin_lock(&my_table_lock); + * pthread_spin_unlock(&my_table_lock); + */ +static inline void ionic_tbl_insert(struct ionic_tbl_root *tbl, + void *val, uint32_t key) +{ + struct ionic_tbl_node *node; + uint32_t node_i = key >> TBL_NODE_SHIFT; + + if (unlikely(key >> TBL_KEY_SHIFT)) { + assert(key >> TBL_KEY_SHIFT == 0); + return; + } + + node = tbl->node[node_i]; + if (!node) + node = tbl->free_node; + + if (unlikely(!node)) { + assert(node != NULL); + return; + } + + /* warning: with NDEBUG the old value will leak */ + assert(node->val[key & TBL_NODE_MASK] == NULL); + + node->val[key & TBL_NODE_MASK] = val; + + if (!tbl->refcount[node_i]) { + tbl->node[node_i] = node; + tbl->free_node = NULL; + } + + ++tbl->refcount[node_i]; +} + +/** + * ionic_tbl_delete() - Delete the value for key in the table + * @tbl: Table root + * @val: Value to insert + * @key: Key to insert + * + * The tbl->free_node must be null when deleting. + * + * Synopsis: + * + * pthread_mutex_lock(&my_table_mut); + * ionic_tbl_free_node(&my_table); + * ionic_tbl_delete(&my_table, key); + * pthread_mutex_unlock(&my_table_mut); + * + * pthread_spin_lock(&my_table_lock); + * pthread_spin_unlock(&my_table_lock); + * free(old_val_at_key); + */ +static inline void ionic_tbl_delete(struct ionic_tbl_root *tbl, uint32_t key) +{ + struct ionic_tbl_node *node; + uint32_t node_i = key >> TBL_NODE_SHIFT; + + if (unlikely(key >> TBL_KEY_SHIFT)) { + assert(key >> TBL_KEY_SHIFT == 0); + return; + } + + node = tbl->node[node_i]; + if (unlikely(!node)) { + assert(node != NULL); + return; + } + + if (unlikely(!node->val[key & TBL_NODE_MASK])) { + assert(node->val[key & TBL_NODE_MASK] != NULL); + return; + } + + node->val[key & TBL_NODE_MASK] = NULL; + + --tbl->refcount[node_i]; + + if (!tbl->refcount[node_i]) { + /* warning: with NDEBUG the old free node will leak */ + assert(node != NULL); + tbl->free_node = node; + tbl->node[node_i] = NULL; + } +} + +#endif /* IONIC_TABLE_H */ diff --git a/providers/ionic/ionic_verbs.c b/providers/ionic/ionic_verbs.c new file mode 100644 index 000000000..589024c71 --- /dev/null +++ b/providers/ionic/ionic_verbs.c @@ -0,0 +1,3037 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* + * Copyright (c) 2018-2025 Advanced Micro Devices, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ionic_fw.h" +#include "ionic.h" + +static void ionic_reserve_sync_cq(struct ionic_ctx *ctx, struct ionic_cq *cq); +static int ionic_poll_cq(struct ibv_cq *ibcq, int nwc, struct ibv_wc *wc); + +#ifdef __x86_64__ +static bool ionic_have_movdir64b; + +#ifndef bit_MOVDIR64B +#define bit_MOVDIR64B BIT(28) +#endif + +static inline bool have_movdir64b(void) +{ +#if (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 4)) + unsigned int ax, bx, cx, dx, count = 0, leaf = 7; + + /* Return highest supported cpuid input value. */ + __asm__ volatile ("cpuid\n\t" + : "=a" (ax), "=b" (bx), "=c" (cx), "=d" (dx) + : "0" (leaf)); + + if (ax == 0 || ax < leaf) + return 0; + + __asm__ volatile ("cpuid\n\t" + : "=a" (ax), "=b" (bx), "=c" (cx), "=d" (dx) + : "0" (leaf), "2" (count)); + + return cx & bit_MOVDIR64B ? 1 : 0; +#else + return 0; +#endif +} + +static __attribute__((constructor)) +void init_ionic_mmio_memcpy(void) +{ + ionic_have_movdir64b = have_movdir64b(); +} + +static inline void ionic_movdir64b_mmio_memcpy_x64_64(void *dst, const void *src) +{ + const struct { char _[64]; } *__src = src; + struct { char _[64]; } *__dst = dst; + + /* + * Caller must guarantee: + * assert(((uintptr_t)dst) % 64 == 0); + * assert(((uintptr_t)src) % 64 == 0); + */ + + /* + * MOVDIR64B %(rdx), rax. + * + * Both __src and __dst must be memory constraints in order to tell the + * compiler that no other memory accesses should be reordered around + * this one. + * + * Also, both must be supplied as lvalues because this tells + * the compiler what the object is (its size) the instruction accesses. + * I.e., not the pointers but what they point to, thus the deref'ing '*'. + */ + asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02" + : "+m" (*__dst) + : "m" (*__src), "a" (__dst), "d" (__src)); +} + +static inline void ionic_movdir64b_mmio_memcpy_x64(void *dst, const void *src, + size_t bytecnt) +{ + /* + * Caller must guarantee: + * assert(bytecnt != 0); + * assert((bytecnt % 64) == 0); + * assert(((uintptr_t)dst) % 64 == 0); + * assert(((uintptr_t)src) % 64 == 0); + */ + + do { + ionic_movdir64b_mmio_memcpy_x64_64(dst, src); + bytecnt -= 64; + src += 64; + dst += 64; + } while (bytecnt > 0); +} +#endif /* #if defined(__x86_64__) */ + +static inline void ionic_mmio_memcpy_x64_64(void *dst, const void *src) +{ +#ifdef __x86_64__ + if (likely(ionic_have_movdir64b)) + ionic_movdir64b_mmio_memcpy_x64_64(dst, src); + else +#endif + mmio_memcpy_x64(dst, src, 64); +} + +static inline void ionic_mmio_memcpy_x64(void *dst, const void *src, size_t bytecnt) +{ +#ifdef __x86_64__ + if (likely(ionic_have_movdir64b)) + ionic_movdir64b_mmio_memcpy_x64(dst, src, bytecnt); + else +#endif + mmio_memcpy_x64(dst, src, bytecnt); +} + +#define ionic_cq_spin_lock(cq) do { \ + if (unlikely(!(cq)->lockfree)) \ + pthread_spin_lock(&(cq)->lock); \ +} while (0) + +#define ionic_cq_spin_trylock(cq) \ + (likely((cq)->lockfree) ? \ + 0 : pthread_spin_trylock(&(cq)->lock)) + +#define ionic_cq_spin_unlock(cq) do { \ + if (unlikely(!(cq)->lockfree)) \ + pthread_spin_unlock(&(cq)->lock); \ +} while (0) + +#define ionic_rq_spin_lock(qp) do { \ + if (unlikely(!(qp)->lockfree)) \ + pthread_spin_lock(&(qp)->rq.lock); \ +} while (0) + +#define ionic_rq_spin_unlock(qp) do { \ + if (unlikely(!(qp)->lockfree)) \ + pthread_spin_unlock(&(qp)->rq.lock); \ +} while (0) + +#define ionic_sq_spin_lock(qp) do { \ + if (unlikely(!(qp)->lockfree)) \ + pthread_spin_lock(&(qp)->sq.lock); \ +} while (0) + +#define ionic_sq_spin_unlock(qp) do { \ + if (unlikely(!(qp)->lockfree)) \ + pthread_spin_unlock(&(qp)->sq.lock); \ +} while (0) + +#define IONIC_OP(version, opname) \ + ((version) < 2 ? IONIC_V1_OP_##opname : IONIC_V2_OP_##opname) + +static int ionic_query_device_ex(struct ibv_context *ibctx, + const struct ibv_query_device_ex_input *input, + struct ibv_device_attr_ex *ex, + size_t ex_size) +{ + struct ibv_device_attr *dev_attr = &ex->orig_attr; + struct ib_uverbs_ex_query_device_resp resp = {}; + size_t resp_size = sizeof(resp); + int rc; + + rc = ibv_cmd_query_device_any(ibctx, input, ex, ex_size, &resp, &resp_size); + if (rc) + return rc; + + rc = ibv_read_sysfs_file(ibctx->device->ibdev_path, "fw_ver", + dev_attr->fw_ver, sizeof(dev_attr->fw_ver)); + if (rc < 0) + dev_attr->fw_ver[0] = 0; + + return 0; +} + +static int ionic_query_port(struct ibv_context *ibctx, uint8_t port, + struct ibv_port_attr *port_attr) +{ + struct ibv_query_port req = {}; + + return ibv_cmd_query_port(ibctx, port, port_attr, + &req, sizeof(req)); +} + +static struct ibv_pd *ionic_alloc_parent_domain(struct ibv_context *context, + struct ibv_parent_domain_init_attr *attr) +{ + struct ionic_pd *pd, *init_pd; + struct ibv_pd *root_ibpd; + int rc; + + if (ibv_check_alloc_parent_domain(attr)) { + rc = errno; + goto err_pd; + } + + if (!check_comp_mask(attr->comp_mask, + IBV_PARENT_DOMAIN_INIT_ATTR_ALLOCATORS | + IBV_PARENT_DOMAIN_INIT_ATTR_PD_CONTEXT)) { + rc = EINVAL; + goto err_pd; + } + + pd = calloc(1, sizeof(*pd)); + if (!pd) { + rc = errno; + goto err_pd; + } + + init_pd = to_ionic_pd(attr->pd); + root_ibpd = ionic_root_ibpd(init_pd); + + ibv_initialize_parent_domain(&pd->ibpd, root_ibpd); + + pd->root_ibpd = root_ibpd; + pd->udma_mask = init_pd->udma_mask; + pd->sq_cmb = init_pd->sq_cmb; + pd->rq_cmb = init_pd->rq_cmb; + + if (attr->comp_mask & IBV_PARENT_DOMAIN_INIT_ATTR_ALLOCATORS) { + pd->alloc = attr->alloc; + pd->free = attr->free; + } + + if (attr->comp_mask & IBV_PARENT_DOMAIN_INIT_ATTR_PD_CONTEXT) + pd->pd_context = attr->pd_context; + + return &pd->ibpd; + +err_pd: + errno = rc; + return NULL; +} + +static struct ibv_pd *ionic_alloc_pd(struct ibv_context *ibctx) +{ + struct ionic_ctx *ctx = to_ionic_ctx(ibctx); + struct ionic_pd *pd; + struct ibv_alloc_pd req = {}; + struct ib_uverbs_alloc_pd_resp resp = {}; + int rc; + + pd = calloc(1, sizeof(*pd)); + if (!pd) { + rc = errno; + goto err_pd; + } + + pd->root_ibpd = &pd->ibpd; + + rc = ibv_cmd_alloc_pd(ibctx, &pd->ibpd, + &req, sizeof(req), + &resp, sizeof(resp)); + if (rc) + goto err_cmd; + + pd->udma_mask = ionic_ctx_udma_mask(ctx); + + pd->sq_cmb = IONIC_CMB_ENABLE; + if (ctx->sq_expdb) + pd->sq_cmb |= IONIC_CMB_EXPDB; + + pd->rq_cmb = IONIC_CMB_ENABLE; + if (ctx->rq_expdb) + pd->rq_cmb |= IONIC_CMB_EXPDB; + + return &pd->ibpd; + +err_cmd: + free(pd); +err_pd: + errno = rc; + return NULL; +} + +static int ionic_dealloc_pd(struct ibv_pd *ibpd) +{ + struct ionic_pd *pd = to_ionic_pd(ibpd); + int rc; + + if (&pd->ibpd == pd->root_ibpd) { + rc = ibv_cmd_dealloc_pd(&pd->ibpd); + if (rc) + return rc; + } + + free(pd); + + return 0; +} + +static struct ibv_mr *ionic_reg_dmabuf_mr(struct ibv_pd *ibpd, uint64_t offset, + size_t length, uint64_t iova, + int fd, int access) +{ + struct ibv_pd *root_ibpd = to_ionic_root_ibpd(ibpd); + struct verbs_mr *vmr; + int rc; + + vmr = calloc(1, sizeof(*vmr)); + if (!vmr) { + rc = errno; + goto err_mr; + } + + rc = ibv_cmd_reg_dmabuf_mr(root_ibpd, offset, length, iova, fd, access, vmr, NULL); + if (rc) + goto err_cmd; + + return &vmr->ibv_mr; + +err_cmd: + free(vmr); +err_mr: + errno = rc; + return NULL; +} + +static struct ibv_mr *ionic_reg_mr(struct ibv_pd *ibpd, + void *addr, + size_t len, + uint64_t hca_va, + int access) +{ + struct ibv_pd *root_ibpd = to_ionic_root_ibpd(ibpd); + struct verbs_mr *vmr; + struct ib_uverbs_reg_mr_resp resp = {}; + struct ibv_reg_mr req = {}; + int rc; + + vmr = calloc(1, sizeof(*vmr)); + if (!vmr) { + rc = errno; + goto err_mr; + } + + rc = ibv_cmd_reg_mr(root_ibpd, addr, len, + hca_va, + access, vmr, + &req, sizeof(req), + &resp, sizeof(resp)); + if (rc) + goto err_cmd; + + return &vmr->ibv_mr; + +err_cmd: + free(vmr); +err_mr: + errno = rc; + return NULL; +} + +static int ionic_dereg_mr(struct verbs_mr *vmr) +{ + int rc; + + rc = ibv_cmd_dereg_mr(vmr); + if (rc) + return rc; + + free(vmr); + + return 0; +} + +static void ionic_vcq_cq_deinit(struct ionic_cq *cq) +{ + if (!cq->vcq) + return; + + cq->vcq = NULL; + ionic_queue_destroy(&cq->q); + pthread_spin_destroy(&cq->lock); +} + +static int ionic_vcq_cq_init1(struct ionic_ctx *ctx, + struct ionic_vcq *vcq, + struct ionic_cq *cq, + struct ibv_cq_init_attr_ex *ex, + struct ionic_pd *pd, + struct ionic_qdesc *req_cq) +{ + int rc; + + cq->vcq = vcq; + + cq->lockfree = false; + pthread_spin_init(&cq->lock, PTHREAD_PROCESS_PRIVATE); + list_head_init(&cq->poll_sq); + list_head_init(&cq->poll_rq); + list_head_init(&cq->flush_sq); + list_head_init(&cq->flush_rq); + + rc = ionic_queue_init(&cq->q, pd, IONIC_PD_TAG_CQ, + ctx->pg_shift, ex->cqe + IONIC_CQ_GRACE, + sizeof(struct ionic_v1_cqe)); + if (rc) + goto err_queue; + + cq->color = true; + cq->reserve = cq->q.mask; + cq->reserve_pending = 0; + + req_cq->addr = (uintptr_t)cq->q.ptr; + req_cq->size = cq->q.size; + req_cq->mask = cq->q.mask; + req_cq->depth_log2 = cq->q.depth_log2; + req_cq->stride_log2 = cq->q.stride_log2; + + return 0; + +err_queue: + pthread_spin_destroy(&cq->lock); + + return rc; +} + +static void ionic_vcq_cq_init2(struct ionic_cq *cq, uint32_t resp_cqid) +{ + cq->cqid = resp_cqid; + + ionic_queue_dbell_init(&cq->q, cq->cqid); +} + +/* + * NOTE: ionic_start_poll, ionic_next_poll and ionic_end_poll provide a + * minimal implementations of the ibv_cq_ex polling mechanism, sufficient to + * make functional tests pass but not performant. The intention of the API is + * that start_poll should take any required locks, next_poll should assume the + * locks are held and end_poll should release them. + */ +static int ionic_start_poll(struct ibv_cq_ex *ibcq_ex, struct ibv_poll_cq_attr *attr) +{ + struct ibv_cq *ibcq = ibv_cq_ex_to_cq(ibcq_ex); + struct ionic_vcq *vcq = to_ionic_vcq(ibcq); + + int rc = ionic_poll_cq(ibcq, 1, &vcq->cur_wc); + + if (rc != 1) /* no completions ready or poll failed */ + return (rc == 0) ? ENOENT : rc; + + ibcq_ex->wr_id = vcq->cur_wc.wr_id; + ibcq_ex->status = vcq->cur_wc.status; + return 0; +} + +static int ionic_next_poll(struct ibv_cq_ex *ibcq_ex) +{ + return ionic_start_poll(ibcq_ex, NULL); +} + +static void ionic_end_poll(struct ibv_cq_ex *ibcq_ex) +{ + /* nothing to do here */ +} + +static enum ibv_wc_opcode ionic_wc_read_opcode(struct ibv_cq_ex *ibcq_ex) +{ + struct ibv_cq *ibcq = ibv_cq_ex_to_cq(ibcq_ex); + struct ionic_vcq *vcq = to_ionic_vcq(ibcq); + + return vcq->cur_wc.opcode; +} + +static uint32_t ionic_wc_read_vendor_err(struct ibv_cq_ex *ibcq_ex) +{ + struct ibv_cq *ibcq = ibv_cq_ex_to_cq(ibcq_ex); + struct ionic_vcq *vcq = to_ionic_vcq(ibcq); + + return vcq->cur_wc.vendor_err; +} + +static uint32_t ionic_wc_read_byte_len(struct ibv_cq_ex *ibcq_ex) +{ + struct ibv_cq *ibcq = ibv_cq_ex_to_cq(ibcq_ex); + struct ionic_vcq *vcq = to_ionic_vcq(ibcq); + + return vcq->cur_wc.byte_len; +} + +static __be32 ionic_wc_read_imm_data(struct ibv_cq_ex *ibcq_ex) +{ + struct ibv_cq *ibcq = ibv_cq_ex_to_cq(ibcq_ex); + struct ionic_vcq *vcq = to_ionic_vcq(ibcq); + + return vcq->cur_wc.imm_data; +} + +static uint32_t ionic_wc_read_qp_num(struct ibv_cq_ex *ibcq_ex) +{ + struct ibv_cq *ibcq = ibv_cq_ex_to_cq(ibcq_ex); + struct ionic_vcq *vcq = to_ionic_vcq(ibcq); + + return vcq->cur_wc.qp_num; +} + +static uint32_t ionic_wc_read_src_qp(struct ibv_cq_ex *ibcq_ex) +{ + struct ibv_cq *ibcq = ibv_cq_ex_to_cq(ibcq_ex); + struct ionic_vcq *vcq = to_ionic_vcq(ibcq); + + return vcq->cur_wc.src_qp; +} + +static unsigned int ionic_wc_read_wc_flags(struct ibv_cq_ex *ibcq_ex) +{ + struct ibv_cq *ibcq = ibv_cq_ex_to_cq(ibcq_ex); + struct ionic_vcq *vcq = to_ionic_vcq(ibcq); + + return vcq->cur_wc.wc_flags; +} + +static uint8_t ionic_wc_read_sl(struct ibv_cq_ex *ibcq_ex) +{ + struct ibv_cq *ibcq = ibv_cq_ex_to_cq(ibcq_ex); + struct ionic_vcq *vcq = to_ionic_vcq(ibcq); + + return vcq->cur_wc.sl; +} + +static uint32_t ionic_wc_read_slid(struct ibv_cq_ex *ibcq_ex) +{ + struct ibv_cq *ibcq = ibv_cq_ex_to_cq(ibcq_ex); + struct ionic_vcq *vcq = to_ionic_vcq(ibcq); + + return vcq->cur_wc.slid; +} + +static uint8_t ionic_wc_read_dlid_path_bits(struct ibv_cq_ex *ibcq_ex) +{ + struct ibv_cq *ibcq = ibv_cq_ex_to_cq(ibcq_ex); + struct ionic_vcq *vcq = to_ionic_vcq(ibcq); + + return vcq->cur_wc.dlid_path_bits; +} + +static struct ibv_cq_ex *ionic_create_cq_ex(struct ibv_context *ibctx, + struct ibv_cq_init_attr_ex *ex) +{ + struct ionic_ctx *ctx = to_ionic_ctx(ibctx); + struct ionic_pd *pd = NULL; + struct ionic_vcq *vcq; + struct uionic_cq req = {}; + struct uionic_cq_resp resp = {}; + int cq_i, rc; + + if (ex->wc_flags & ~IONIC_CQ_SUPPORTED_WC_FLAGS) { + rc = ENOTSUP; + goto err; + } + + if (ex->cqe < 1 || ex->cqe + IONIC_CQ_GRACE > 0xffff) { + rc = EINVAL; + goto err; + } + + vcq = calloc(1, sizeof(*vcq)); + if (!vcq) { + rc = ENOMEM; + goto err; + } + + if (ex->comp_mask & IBV_CQ_INIT_ATTR_MASK_PD) { + pd = to_ionic_pd(ex->parent_domain); + vcq->udma_mask = pd->udma_mask; + } else { + vcq->udma_mask = ionic_ctx_udma_mask(ctx); + } + + for (cq_i = 0; cq_i < ctx->udma_count; ++cq_i) { + if (!(vcq->udma_mask & BIT(cq_i))) + continue; + + rc = ionic_vcq_cq_init1(ctx, vcq, &vcq->cq[cq_i], ex, pd, &req.cq[cq_i]); + if (rc) + goto err_cq; + } + + req.udma_mask = vcq->udma_mask; + + rc = ibv_cmd_create_cq(ibctx, ex->cqe, ex->channel, + ex->comp_vector, &vcq->vcq.cq, + &req.ibv_cmd, sizeof(req), + &resp.ibv_resp, sizeof(resp)); + if (rc) + goto err_cmd; + + if (resp.udma_mask != vcq->udma_mask) { + rc = EINVAL; + goto err_udma; + } + + for (cq_i = 0; cq_i < ctx->udma_count; ++cq_i) { + if (!(vcq->udma_mask & BIT(cq_i))) + continue; + + ionic_vcq_cq_init2(&vcq->cq[cq_i], resp.cqid[cq_i]); + } + + vcq->vcq.cq_ex.start_poll = ionic_start_poll; + vcq->vcq.cq_ex.next_poll = ionic_next_poll; + vcq->vcq.cq_ex.end_poll = ionic_end_poll; + + vcq->vcq.cq_ex.read_opcode = ionic_wc_read_opcode; + vcq->vcq.cq_ex.read_vendor_err = ionic_wc_read_vendor_err; + vcq->vcq.cq_ex.read_wc_flags = ionic_wc_read_wc_flags; + + if (ex->wc_flags & IBV_WC_EX_WITH_BYTE_LEN) + vcq->vcq.cq_ex.read_byte_len = ionic_wc_read_byte_len; + if (ex->wc_flags & IBV_WC_EX_WITH_IMM) + vcq->vcq.cq_ex.read_imm_data = ionic_wc_read_imm_data; + if (ex->wc_flags & IBV_WC_EX_WITH_QP_NUM) + vcq->vcq.cq_ex.read_qp_num = ionic_wc_read_qp_num; + if (ex->wc_flags & IBV_WC_EX_WITH_SRC_QP) + vcq->vcq.cq_ex.read_src_qp = ionic_wc_read_src_qp; + if (ex->wc_flags & IBV_WC_EX_WITH_SL) + vcq->vcq.cq_ex.read_sl = ionic_wc_read_sl; + if (ex->wc_flags & IBV_WC_EX_WITH_SLID) + vcq->vcq.cq_ex.read_slid = ionic_wc_read_slid; + if (ex->wc_flags & IBV_WC_EX_WITH_DLID_PATH_BITS) + vcq->vcq.cq_ex.read_dlid_path_bits = ionic_wc_read_dlid_path_bits; + + return &vcq->vcq.cq_ex; + +err_udma: + (void)ibv_cmd_destroy_cq(&vcq->vcq.cq); +err_cmd: + while (cq_i) { + --cq_i; + if (!(vcq->udma_mask & BIT(cq_i))) + continue; + ionic_vcq_cq_deinit(&vcq->cq[cq_i]); +err_cq: + ; + } + free(vcq); +err: + errno = rc; + return NULL; +} + +static struct ibv_cq *ionic_create_cq(struct ibv_context *ibctx, int ncqe, + struct ibv_comp_channel *channel, + int vec) +{ + struct ibv_cq_init_attr_ex ex = { + .cqe = ncqe, + .channel = channel, + .comp_vector = vec, + }; + struct ibv_cq_ex *ibcq; + + if (ncqe < 0) { + errno = EINVAL; + return NULL; + } + + ibcq = ionic_create_cq_ex(ibctx, &ex); + + return ibv_cq_ex_to_cq(ibcq); +} + +static int ionic_destroy_cq(struct ibv_cq *ibcq) +{ + struct ionic_ctx *ctx = to_ionic_ctx(ibcq->context); + struct ionic_vcq *vcq = to_ionic_vcq(ibcq); + int cq_i, rc; + + rc = ibv_cmd_destroy_cq(ibcq); + if (rc) + return rc; + + for (cq_i = ctx->udma_count; cq_i;) { + --cq_i; + + if (!(vcq->udma_mask & BIT(cq_i))) + continue; + + ionic_vcq_cq_deinit(&vcq->cq[cq_i]); + } + + free(vcq); + + return 0; +} + +static int ionic_flush_recv(struct ionic_qp *qp, struct ibv_wc *wc) +{ + struct ionic_rq_meta *meta; + struct ionic_v1_wqe *wqe; + struct ionic_ctx *ctx; + + if (!qp->rq.flush) + return 0; + + if (ionic_queue_empty(&qp->rq.queue)) + return 0; + + wqe = ionic_queue_at_cons(&qp->rq.queue); + ctx = to_ionic_ctx(qp->vqp.qp.context); + + /* wqe_id must be a valid queue index */ + if (unlikely(wqe->base.wqe_id >> qp->rq.queue.depth_log2)) { + verbs_err(&ctx->vctx, "invalid id %#lx", + (unsigned long)wqe->base.wqe_id); + return -EIO; + } + + /* wqe_id must indicate a request that is outstanding */ + meta = &qp->rq.meta[wqe->base.wqe_id]; + if (unlikely(meta->next != IONIC_META_POSTED)) { + verbs_err(&ctx->vctx, "wqe not posted %#lx", + (unsigned long)wqe->base.wqe_id); + return -EIO; + } + + ionic_queue_consume(&qp->rq.queue); + + memset(wc, 0, sizeof(*wc)); + + wc->status = IBV_WC_WR_FLUSH_ERR; + wc->wr_id = meta->wrid; + wc->qp_num = qp->qpid; + + meta->next = qp->rq.meta_head; + qp->rq.meta_head = meta; + + return 1; +} + +static int ionic_flush_recv_many(struct ionic_qp *qp, + struct ibv_wc *wc, int nwc) +{ + int rc = 0, npolled = 0; + + while (npolled < nwc) { + rc = ionic_flush_recv(qp, wc + npolled); + if (rc <= 0) + break; + + npolled += rc; + } + + return npolled ?: rc; +} + +static int ionic_flush_send(struct ionic_qp *qp, struct ibv_wc *wc) +{ + struct ionic_sq_meta *meta; + + if (!qp->sq.flush) + return 0; + + if (ionic_queue_empty(&qp->sq.queue)) + return 0; + + meta = &qp->sq.meta[qp->sq.queue.cons]; + + ionic_queue_consume(&qp->sq.queue); + + memset(wc, 0, sizeof(*wc)); + + wc->status = IBV_WC_WR_FLUSH_ERR; + wc->wr_id = meta->wrid; + wc->qp_num = qp->qpid; + + return 1; +} + +static int ionic_flush_send_many(struct ionic_qp *qp, + struct ibv_wc *wc, int nwc) +{ + int rc = 0, npolled = 0; + + while (npolled < nwc) { + rc = ionic_flush_send(qp, wc + npolled); + if (rc <= 0) + break; + + npolled += rc; + } + + return npolled ?: rc; +} + +static int ionic_poll_recv(struct ionic_ctx *ctx, struct ionic_cq *cq, + struct ionic_qp *cqe_qp, struct ionic_v1_cqe *cqe, + struct ibv_wc *wc) +{ + struct ionic_qp *qp = NULL; + struct ionic_rq_meta *meta; + uint32_t src_qpn, st_len; + uint16_t vlan_tag; + uint8_t op; + + if (cqe_qp->rq.flush) + return 0; + + qp = cqe_qp; + + st_len = be32toh(cqe->status_length); + + /* ignore wqe_id in case of flush error */ + if (ionic_v1_cqe_error(cqe) && st_len == IONIC_STS_WQE_FLUSHED_ERR) { + cqe_qp->rq.flush = true; + cq->flush = true; + list_del(&cqe_qp->cq_flush_rq); + list_add_tail(&cq->flush_rq, &cqe_qp->cq_flush_rq); + + /* posted recvs (if any) flushed by ionic_flush_recv */ + return 0; + } + + /* there had better be something in the recv queue to complete */ + if (ionic_queue_empty(&qp->rq.queue)) { + verbs_err(&ctx->vctx, "rq is empty %u", qp->qpid); + return -EIO; + } + + /* wqe_id must be a valid queue index */ + if (unlikely(cqe->recv.wqe_id >> qp->rq.queue.depth_log2)) { + verbs_err(&ctx->vctx, "invalid id %#lx", + (unsigned long)cqe->recv.wqe_id); + return -EIO; + } + + /* wqe_id must indicate a request that is outstanding */ + meta = &qp->rq.meta[qp->rq.meta_idx[cqe->recv.wqe_id]]; + if (unlikely(meta->next != IONIC_META_POSTED)) { + verbs_err(&ctx->vctx, "wqe is not posted for idx %lu meta_idx %u qpid %u rq.prod %u rq.cons %u cqid %u", + (unsigned long)cqe->recv.wqe_id, + qp->rq.meta_idx[cqe->recv.wqe_id], + qp->qpid, qp->rq.queue.prod, + qp->rq.queue.cons, cq->cqid); + return -EIO; + } + + meta->next = qp->rq.meta_head; + qp->rq.meta_head = meta; + + memset(wc, 0, sizeof(*wc)); + + wc->wr_id = meta->wrid; + wc->qp_num = cqe_qp->qpid; + + if (ionic_v1_cqe_error(cqe)) { + wc->vendor_err = st_len; + wc->status = ionic_to_ibv_status(st_len); + + cqe_qp->rq.flush = true; + cq->flush = true; + list_del(&cqe_qp->cq_flush_rq); + list_add_tail(&cq->flush_rq, &cqe_qp->cq_flush_rq); + + verbs_err(&ctx->vctx, "cqe with error %u for %#x (recv), qpid %u cqid %u", + wc->status, be32toh(cqe->send.msg_msn), + qp->qpid, cq->cqid); + + goto out; + } + + wc->vendor_err = 0; + wc->status = IBV_WC_SUCCESS; + + src_qpn = be32toh(cqe->recv.src_qpn_op); + op = src_qpn >> IONIC_V1_CQE_RECV_OP_SHIFT; + + src_qpn &= IONIC_V1_CQE_RECV_QPN_MASK; + op &= IONIC_V1_CQE_RECV_OP_MASK; + + wc->opcode = IBV_WC_RECV; + switch (op) { + case IONIC_V1_CQE_RECV_OP_RDMA_IMM: + wc->opcode = IBV_WC_RECV_RDMA_WITH_IMM; + SWITCH_FALLTHROUGH; + case IONIC_V1_CQE_RECV_OP_SEND_IMM: + wc->wc_flags |= IBV_WC_WITH_IMM; + wc->imm_data = cqe->recv.imm_data_rkey; /* be32 in wc */ + break; + case IONIC_V1_CQE_RECV_OP_SEND_INV: + wc->wc_flags |= IBV_WC_WITH_INV; + wc->invalidated_rkey = be32toh(cqe->recv.imm_data_rkey); + } + + wc->byte_len = st_len; + wc->src_qp = src_qpn; + + if (qp->vqp.qp.qp_type == IBV_QPT_UD) { + wc->wc_flags |= IBV_WC_GRH; + + /* vlan_tag in cqe will be valid from dpath even if no vlan */ + vlan_tag = be16toh(cqe->recv.vlan_tag); + wc->sl = vlan_tag >> 13; /* 802.1q PCP */ + } + + wc->pkey_index = 0; + +out: + verbs_debug(&ctx->vctx, "poll cq %u qp %u cons %u st %u wrid %" PRIx64 " op %u len %u", + cq->cqid, qp->qpid, qp->rq.queue.cons, wc->status, + meta->wrid, wc->opcode, st_len); + cq->reserve_pending++; + ionic_queue_consume(&qp->rq.queue); + + return 1; +} + +static bool ionic_peek_send(struct ionic_qp *qp) +{ + struct ionic_sq_meta *meta; + + if (qp->sq.flush) + return 0; + + /* completed all send queue requests? */ + if (ionic_queue_empty(&qp->sq.queue)) + return false; + + meta = &qp->sq.meta[qp->sq.queue.cons]; + + /* waiting for remote completion? */ + if (meta->remote && meta->seq == qp->sq.msn_cons) + return false; + + /* waiting for local completion? */ + if (!meta->remote && !meta->local_comp) + return false; + + return true; +} + +static int ionic_poll_send(struct ionic_ctx *ctx, + struct ionic_cq *cq, + struct ionic_qp *qp, + struct ibv_wc *wc) +{ + struct ionic_sq_meta *meta; + + if (qp->sq.flush) + return 0; + + do { + /* completed all send queue requests? */ + if (ionic_queue_empty(&qp->sq.queue)) + goto out_empty; + + meta = &qp->sq.meta[qp->sq.queue.cons]; + + /* waiting for remote completion? */ + if (meta->remote && meta->seq == qp->sq.msn_cons) + goto out_empty; + + /* waiting for local completion? */ + if (!meta->remote && !meta->local_comp) + goto out_empty; + + verbs_debug(&ctx->vctx, "poll cq %u qp %u cons %u st %u wr %" PRIx64 " op %u l %u", + cq->cqid, qp->qpid, qp->sq.queue.cons, meta->ibsts, + meta->wrid, meta->ibop, meta->len); + cq->reserve_pending++; + ionic_queue_consume(&qp->sq.queue); + + /* produce wc only if signaled or error status */ + } while (!meta->signal && meta->ibsts == IBV_WC_SUCCESS); + + memset(wc, 0, sizeof(*wc)); + + wc->status = meta->ibsts; + wc->wr_id = meta->wrid; + wc->qp_num = qp->qpid; + + if (meta->ibsts == IBV_WC_SUCCESS) { + wc->byte_len = meta->len; + wc->opcode = meta->ibop; + } else { + wc->vendor_err = meta->len; + + qp->sq.flush = true; + cq->flush = true; + list_del(&qp->cq_flush_sq); + list_add_tail(&cq->flush_sq, &qp->cq_flush_sq); + } + + return 1; + +out_empty: + if (qp->sq.flush_rcvd) { + qp->sq.flush = true; + cq->flush = true; + list_del(&qp->cq_flush_sq); + list_add_tail(&cq->flush_sq, &qp->cq_flush_sq); + } + return 0; +} + +static int ionic_poll_send_many(struct ionic_ctx *ctx, + struct ionic_cq *cq, + struct ionic_qp *qp, + struct ibv_wc *wc, int nwc) +{ + int rc = 0, npolled = 0; + + while (npolled < nwc) { + rc = ionic_poll_send(ctx, cq, qp, wc + npolled); + if (rc <= 0) + break; + + npolled += rc; + } + + return npolled ?: rc; +} + +static int ionic_validate_cons(uint16_t prod, uint16_t cons, + uint16_t comp, uint16_t mask) +{ + if (((prod - cons) & mask) <= ((comp - cons) & mask)) + return -EIO; + + return 0; +} + +static int ionic_comp_msn(struct ionic_ctx *ctx, + struct ionic_qp *qp, + struct ionic_v1_cqe *cqe) +{ + struct ionic_sq_meta *meta; + uint16_t cqe_seq, cqe_idx; + int rc; + + if (qp->sq.flush) + return 0; + + cqe_seq = be32toh(cqe->send.msg_msn) & qp->sq.queue.mask; + + rc = ionic_validate_cons(qp->sq.msn_prod, + qp->sq.msn_cons, + cqe_seq - 1, + qp->sq.queue.mask); + if (rc) { + struct ionic_cq *cq = + to_ionic_vcq_cq(qp->vqp.qp.send_cq, qp->udma_idx); + + verbs_err(&ctx->vctx, "wqe is not posted for %#x (msn), qpid %u cqid %u seq %u for prod %u cons %u\n", + be32toh(cqe->send.msg_msn), qp->qpid, cq->cqid, + cqe_seq, qp->sq.msn_prod, qp->sq.msn_cons); + return rc; + } + + qp->sq.msn_cons = cqe_seq; + + if (ionic_v1_cqe_error(cqe)) { + struct ionic_cq *cq = + to_ionic_vcq_cq(qp->vqp.qp.send_cq, qp->udma_idx); + + cqe_idx = qp->sq.msn_idx[(cqe_seq - 1) & qp->sq.queue.mask]; + + meta = &qp->sq.meta[cqe_idx]; + meta->len = be32toh(cqe->status_length); + meta->ibsts = ionic_to_ibv_status(meta->len); + + verbs_err(&ctx->vctx, "cqe with error %u for %#x (msn), qpid %u cqid %u", + meta->ibsts, be32toh(cqe->send.msg_msn), + qp->qpid, cq->cqid); + } + + return 0; +} + +static int ionic_comp_npg(struct ionic_ctx *ctx, + struct ionic_qp *qp, + struct ionic_v1_cqe *cqe) +{ + struct ionic_sq_meta *meta; + uint16_t cqe_idx; + uint32_t st_len; + + if (qp->sq.flush) + return 0; + + st_len = be32toh(cqe->status_length); + + if (ionic_v1_cqe_error(cqe) && st_len == IONIC_STS_WQE_FLUSHED_ERR) { + /* Flush cqe does not consume a wqe on the device, and maybe + * no such work request is posted. + * + * The driver should begin flushing after the last indicated + * normal or error completion. Here, only set a hint that the + * flush request was indicated. In poll_send, if nothing more + * can be polled normally, then begin flushing. + */ + qp->sq.flush_rcvd = true; + return 0; + } + + cqe_idx = cqe->send.npg_wqe_id & qp->sq.queue.mask; + meta = &qp->sq.meta[cqe_idx]; + meta->local_comp = true; + + if (ionic_v1_cqe_error(cqe)) { + struct ionic_cq *cq = + to_ionic_vcq_cq(qp->vqp.qp.send_cq, qp->udma_idx); + + meta->len = st_len; + meta->ibsts = ionic_to_ibv_status(st_len); + meta->remote = false; + verbs_err(&ctx->vctx, "cqe with error %s for %#x (npg), qpid %u cqid %u", + ibv_wc_status_str(meta->ibsts), + be32toh(cqe->send.msg_msn), + qp->qpid, cq->cqid); + } + + return 0; +} + +static bool ionic_next_cqe(struct ionic_ctx *ctx, + struct ionic_cq *cq, + struct ionic_v1_cqe **cqe) +{ + struct ionic_v1_cqe *qcqe = ionic_queue_at_prod(&cq->q); + + if (unlikely(cq->color != ionic_v1_cqe_color(qcqe))) + return false; + + udma_from_device_barrier(); + + ionic_dbg_xdump(ctx, "cqe", qcqe, 1u << cq->q.stride_log2); + *cqe = qcqe; + + return true; +} + +static void ionic_clean_cq(struct ionic_cq *cq, uint32_t qpid) +{ + struct ionic_v1_cqe *qcqe; + int prod, qtf, qid; + bool color; + + color = cq->color; + prod = cq->q.prod; + qcqe = ionic_queue_at(&cq->q, prod); + + while (color == ionic_v1_cqe_color(qcqe)) { + qtf = ionic_v1_cqe_qtf(qcqe); + qid = ionic_v1_cqe_qtf_qid(qtf); + + if (qid == qpid) + ionic_v1_cqe_clean(qcqe); + + prod = ionic_queue_next(&cq->q, prod); + qcqe = ionic_queue_at(&cq->q, prod); + color = ionic_color_wrap(prod, color); + } +} + +static void ionic_reserve_sync_cq(struct ionic_ctx *ctx, struct ionic_cq *cq) +{ + if (!ionic_queue_empty(&cq->q)) { + cq->reserve += cq->reserve_pending; + cq->reserve_pending = 0; + cq->q.cons = cq->q.prod; + + verbs_debug(&ctx->vctx, "dbell cq %u val %" PRIx64 " rsv %d", + cq->cqid, ionic_queue_dbell_val(&cq->q), + cq->reserve); + ionic_dbell_ring(&ctx->dbpage[ctx->cq_qtype], + ionic_queue_dbell_val(&cq->q)); + } +} + +static void ionic_arm_cq(struct ionic_ctx *ctx, struct ionic_cq *cq) +{ + uint64_t dbell_val = cq->q.dbell; + + if (cq->deferred_arm_sol_only) { + cq->arm_sol_prod = ionic_queue_next(&cq->q, cq->arm_sol_prod); + dbell_val |= cq->arm_sol_prod | IONIC_DBELL_RING_SONLY; + } else { + cq->arm_any_prod = ionic_queue_next(&cq->q, cq->arm_any_prod); + dbell_val |= cq->arm_any_prod | IONIC_DBELL_RING_ARM; + } + + verbs_debug(&ctx->vctx, "dbell cq %u val %" PRIx64 " (%s)", + cq->cqid, dbell_val, + cq->deferred_arm_sol_only ? "sonly" : "arm"); + ionic_dbell_ring(&ctx->dbpage[ctx->cq_qtype], dbell_val); +} + +static void ionic_reserve_cq(struct ionic_ctx *ctx, struct ionic_cq *cq, + int spend) +{ + cq->reserve -= spend; + + if (cq->reserve <= 0 || cq->deferred_arm) + ionic_reserve_sync_cq(ctx, cq); + + if (cq->deferred_arm) { + ionic_arm_cq(ctx, cq); + cq->deferred_arm = false; + } +} + +static int ionic_poll_vcq_cq(struct ionic_ctx *ctx, struct ionic_cq *cq, + int nwc, struct ibv_wc *wc) +{ + struct ionic_qp *qp, *qp_next; + struct ionic_v1_cqe *cqe; + uint32_t qtf, qid; + uint8_t type; + bool peek; + int rc = 0, npolled = 0; + uint16_t old_prod; + + if (nwc < 1) + return 0; + + ionic_cq_spin_lock(cq); + ++cq->cqseq; + + old_prod = cq->q.prod; + + /* poll already indicated work completions for send queue */ + + list_for_each_safe(&cq->poll_sq, qp, qp_next, cq_poll_sq) { + if (npolled == nwc) + goto out; + + ionic_sq_spin_lock(qp); + rc = ionic_poll_send_many(ctx, cq, qp, wc + npolled, nwc - npolled); + peek = ionic_peek_send(qp); + ionic_sq_spin_unlock(qp); + + if (rc < 0) + goto out; + + npolled += rc; + + if (!peek) + list_del_init(&qp->cq_poll_sq); + } + + /* poll for more work completions */ + + while (likely(ionic_next_cqe(ctx, cq, &cqe))) { + if (npolled == nwc) + goto out; + + qtf = ionic_v1_cqe_qtf(cqe); + qid = ionic_v1_cqe_qtf_qid(qtf); + type = ionic_v1_cqe_qtf_type(qtf); + + qp = ionic_tbl_lookup(&ctx->qp_tbl, qid); + + if (unlikely(!qp)) { + verbs_err(&ctx->vctx, "cq %d pos %d missing qp for qid %#x", + cq->cqid, cq->q.prod, qid); + goto cq_next; + } + + switch (type) { + case IONIC_V1_CQE_TYPE_RECV: + ionic_rq_spin_lock(qp); + rc = ionic_poll_recv(ctx, cq, qp, cqe, wc + npolled); + ionic_rq_spin_unlock(qp); + + if (rc < 0) + goto out; + + npolled += rc; + break; + + case IONIC_V1_CQE_TYPE_SEND_MSN: + ionic_sq_spin_lock(qp); + rc = ionic_comp_msn(ctx, qp, cqe); + if (!rc) { + rc = ionic_poll_send_many(ctx, cq, qp, + wc + npolled, + nwc - npolled); + peek = ionic_peek_send(qp); + } + ionic_sq_spin_unlock(qp); + + if (rc < 0) + goto out; + + npolled += rc; + + if (peek) { + list_del(&qp->cq_poll_sq); + list_add_tail(&cq->poll_sq, &qp->cq_poll_sq); + } + + break; + + case IONIC_V1_CQE_TYPE_SEND_NPG: + ionic_sq_spin_lock(qp); + rc = ionic_comp_npg(ctx, qp, cqe); + if (!rc) { + rc = ionic_poll_send_many(ctx, cq, qp, + wc + npolled, + nwc - npolled); + peek = ionic_peek_send(qp); + } + ionic_sq_spin_unlock(qp); + + if (rc < 0) + goto out; + + npolled += rc; + + if (peek) { + list_del(&qp->cq_poll_sq); + list_add_tail(&cq->poll_sq, &qp->cq_poll_sq); + } + + break; + + case IONIC_V1_CQE_TYPE_RECV_INDIR: + list_del(&qp->cq_poll_rq); + list_add_tail(&cq->poll_rq, &qp->cq_poll_rq); + break; + + default: + verbs_err(&ctx->vctx, "unexpected cqe type %u", type); + rc = -EIO; + goto out; + } + +cq_next: + ionic_queue_produce(&cq->q); + cq->color = ionic_color_wrap(cq->q.prod, cq->color); + } + + /* lastly, flush send and recv queues */ + if (likely(!cq->flush)) + goto out; + + cq->flush = false; + + list_for_each_safe(&cq->flush_sq, qp, qp_next, cq_flush_sq) { + if (npolled == nwc) + goto out; + + ionic_sq_spin_lock(qp); + rc = ionic_flush_send_many(qp, wc + npolled, nwc - npolled); + ionic_sq_spin_unlock(qp); + + if (rc > 0) + npolled += rc; + + if (npolled < nwc) + list_del_init(&qp->cq_flush_sq); + else + cq->flush = true; + } + + list_for_each_safe(&cq->flush_rq, qp, qp_next, cq_flush_rq) { + if (npolled == nwc) + goto out; + + ionic_rq_spin_lock(qp); + rc = ionic_flush_recv_many(qp, wc + npolled, nwc - npolled); + ionic_rq_spin_unlock(qp); + + if (rc > 0) + npolled += rc; + + if (npolled < nwc) + list_del_init(&qp->cq_flush_rq); + else + cq->flush = true; + } + +out: + ionic_reserve_cq(ctx, cq, 0); + + old_prod = (cq->q.prod - old_prod) & cq->q.mask; + + ionic_cq_spin_unlock(cq); + + return npolled ?: rc; +} + +/* + * Note about rc: (noted here because poll is different) + * + * Functions without "poll" in the name, if they return an integer, return + * zero on success, or a positive error number. Functions returning a + * pointer return NULL on error and set errno to a positive error number. + * + * Functions with "poll" in the name return negative error numbers, or + * greater or equal to zero number of completions on success. + */ +static int ionic_poll_cq(struct ibv_cq *ibcq, int nwc, struct ibv_wc *wc) +{ + struct ionic_ctx *ctx = to_ionic_ctx(ibcq->context); + struct ionic_vcq *vcq = to_ionic_vcq(ibcq); + int rc_tmp, rc = 0, npolled = 0; + int cq_i, cq_x, cq_ix; + + cq_x = vcq->poll_idx; + + vcq->poll_idx ^= ctx->udma_count - 1; + + for (cq_i = 0; npolled < nwc && cq_i < ctx->udma_count; ++cq_i) { + cq_ix = cq_i ^ cq_x; + + if (!(vcq->udma_mask & BIT(cq_ix))) + continue; + + rc_tmp = ionic_poll_vcq_cq(ctx, &vcq->cq[cq_ix], + nwc - npolled, wc + npolled); + + if (rc_tmp >= 0) + npolled += rc_tmp; + else if (!rc) + rc = rc_tmp; + } + + return npolled ?: rc; +} + +static void ionic_req_notify_vcq_cq(struct ionic_cq *cq, int solicited_only) +{ + ionic_cq_spin_lock(cq); + + if (cq->deferred_arm && !cq->deferred_arm_sol_only) + solicited_only = 0; /* do not downgrade scheduled ARM to SARM */ + + cq->deferred_arm = true; + cq->deferred_arm_sol_only = (bool)solicited_only; + + ionic_cq_spin_unlock(cq); +} + +static int ionic_req_notify_cq(struct ibv_cq *ibcq, int solicited_only) +{ + struct ionic_ctx *ctx = to_ionic_ctx(ibcq->context); + struct ionic_vcq *vcq = to_ionic_vcq(ibcq); + int cq_i; + + if (!vcq->vcq.cq.channel) + return -ENOENT; + + for (cq_i = 0; cq_i < ctx->udma_count; ++cq_i) { + if (!(vcq->udma_mask & BIT(cq_i))) + continue; + + ionic_req_notify_vcq_cq(&vcq->cq[cq_i], solicited_only); + } + + return 0; +} + +static bool ionic_expdb_wqe_size_supported(struct ionic_ctx *ctx, + uint32_t wqe_size) +{ + switch (wqe_size) { + case 64: return ctx->expdb_mask & IONIC_EXPDB_64; + case 128: return ctx->expdb_mask & IONIC_EXPDB_128; + case 256: return ctx->expdb_mask & IONIC_EXPDB_256; + case 512: return ctx->expdb_mask & IONIC_EXPDB_512; + } + + return false; +} + +static int ionic_set_cmb(struct ionic_ctx *ctx, struct ionic_queue *queue, + uint8_t cmb, uint64_t cmb_offset, uint8_t qtype, + void **cmb_ptr) +{ + uint64_t db_base, db_data; + uint16_t wqe_sz, pos; + __le64 *db_addr; + void *db_ptr; + + if (!(cmb & IONIC_CMB_ENABLE)) { + *cmb_ptr = NULL; + return 0; + } + + *cmb_ptr = ionic_map_device(queue->size, ctx->vctx.context.cmd_fd, + cmb_offset); + if (!*cmb_ptr) + return errno; + + if (cmb & IONIC_CMB_EXPDB) { + /* Pre-fill express doorbells into our buffer */ + wqe_sz = 1 << queue->stride_log2; + db_ptr = queue->ptr + wqe_sz - IONIC_EXP_DBELL_SZ; + + /* Assume ring 0 */ + db_base = ((uint64_t)qtype << 48) | queue->dbell | + ((0x03 | 0x8) << 19); /* SCHED_SET | PICI_PISET */ + + for (pos = 0; pos <= queue->mask; pos++, db_ptr += wqe_sz) { + db_addr = (__le64 *)db_ptr; + db_data = db_base | ((pos + 1) & queue->mask); + *db_addr = htole64(db_data); + } + } + + return 0; +} + +static int ionic_qp_sq_init(struct ionic_ctx *ctx, struct ionic_qp *qp, struct ionic_pd *pd, + int max_wr, int max_sge, int max_data) +{ + uint32_t wqe_size; + int rc; + + if (!qp->has_sq) + return 0; + + if (max_wr < 0 || max_wr > 0xffff) + return EINVAL; + if (max_sge < 0) + return EINVAL; + if (max_sge > ionic_v1_send_wqe_max_sge(ctx->max_stride, 0, false)) + return EINVAL; + if (max_data < 0) + return EINVAL; + if (max_data > ionic_v1_send_wqe_max_data(ctx->max_stride, false)) + return EINVAL; + + qp->sq.spec = ionic_v1_use_spec_sge(max_sge, ctx->spec); + + if (qp->sq.cmb & IONIC_CMB_EXPDB) { + wqe_size = ionic_v1_send_wqe_min_size(max_sge, max_data, + qp->sq.spec, true); + + if (!ionic_expdb_wqe_size_supported(ctx, wqe_size)) + qp->sq.cmb &= ~IONIC_CMB_EXPDB; + } + + if (!(qp->sq.cmb & IONIC_CMB_EXPDB)) + wqe_size = ionic_v1_send_wqe_min_size(max_sge, max_data, + qp->sq.spec, false); + + rc = ionic_queue_init(&qp->sq.queue, pd, IONIC_PD_TAG_SQ, + ctx->pg_shift, max_wr, wqe_size); + if (rc) + goto err_sq; + + qp->sq.cmb_ptr = NULL; + qp->sq.cmb_prod = 0; + qp->sq.color = true; + + qp->sq.meta = calloc((uint32_t)qp->sq.queue.mask + 1, + sizeof(*qp->sq.meta)); + if (!qp->sq.meta) { + rc = ENOMEM; + goto err_sq_meta; + } + + qp->sq.msn_idx = calloc((uint32_t)qp->sq.queue.mask + 1, + sizeof(*qp->sq.msn_idx)); + if (!qp->sq.msn_idx) { + rc = ENOMEM; + goto err_sq_msn; + } + + return 0; + +err_sq_msn: + free(qp->sq.meta); +err_sq_meta: + ionic_queue_destroy(&qp->sq.queue); +err_sq: + return rc; +} + +static void ionic_qp_sq_destroy(struct ionic_qp *qp) +{ + if (!qp->has_sq) + return; + + free(qp->sq.msn_idx); + free(qp->sq.meta); + ionic_queue_destroy(&qp->sq.queue); +} + +static int ionic_qp_rq_init(struct ionic_ctx *ctx, struct ionic_qp *qp, struct ionic_pd *pd, + int max_wr, int max_sge) +{ + uint32_t wqe_size; + uint64_t pd_tag; + int rc, i; + + if (!qp->has_rq) + return 0; + + if (max_wr < 0 || max_wr > 0xffff) + return EINVAL; + if (max_sge < 0) + return EINVAL; + if (max_sge > ionic_v1_recv_wqe_max_sge(ctx->max_stride, 0, false)) + return EINVAL; + + pd_tag = IONIC_PD_TAG_RQ; + qp->rq.spec = ionic_v1_use_spec_sge(max_sge, ctx->spec); + + if (qp->rq.cmb & IONIC_CMB_EXPDB) { + wqe_size = ionic_v1_recv_wqe_min_size(max_sge, qp->rq.spec, true); + + if (!ionic_expdb_wqe_size_supported(ctx, wqe_size)) + qp->rq.cmb &= ~IONIC_CMB_EXPDB; + } + + if (!(qp->rq.cmb & IONIC_CMB_EXPDB)) + wqe_size = ionic_v1_recv_wqe_min_size(max_sge, qp->rq.spec, false); + + rc = ionic_queue_init(&qp->rq.queue, pd, pd_tag, ctx->pg_shift, max_wr, wqe_size); + if (rc) + goto err_rq; + + qp->rq.cmb_ptr = NULL; + qp->rq.cmb_prod = 0; + + qp->rq.meta = calloc((uint32_t)qp->rq.queue.mask + 1, + sizeof(*qp->rq.meta)); + if (!qp->rq.meta) { + rc = ENOMEM; + goto err_rq_meta; + } + + for (i = 0; i < qp->rq.queue.mask; ++i) + qp->rq.meta[i].next = &qp->rq.meta[i + 1]; + + qp->rq.meta[i].next = IONIC_META_LAST; + qp->rq.meta_head = &qp->rq.meta[0]; + + qp->rq.meta_idx = calloc((uint32_t)qp->rq.queue.mask + 1, + sizeof(*qp->rq.meta_idx)); + if (!qp->rq.meta_idx) { + rc = ENOMEM; + goto err_rq_meta_idx; + } + + return 0; + +err_rq_meta_idx: + free(qp->rq.meta); +err_rq_meta: + ionic_queue_destroy(&qp->rq.queue); +err_rq: + return rc; +} + +static void ionic_qp_rq_destroy(struct ionic_qp *qp) +{ + if (!qp->has_rq) + return; + + free(qp->rq.meta_idx); + free(qp->rq.meta); + ionic_queue_destroy(&qp->rq.queue); +} + +static struct ibv_qp *ionic_create_qp_ex(struct ibv_context *ibctx, + struct ibv_qp_init_attr_ex *ex) +{ + struct ionic_ctx *ctx = to_ionic_ctx(ibctx); + struct ionic_pd *pd = to_ionic_pd(ex->pd); + struct uionic_qp_resp resp; + struct uionic_qp req; + struct ionic_qp *qp; + struct ionic_cq *cq; + int rc; + + qp = calloc(1, sizeof(*qp)); + if (!qp) { + rc = ENOMEM; + goto err_qp; + } + + qp->vqp.qp.qp_type = ex->qp_type; + qp->has_sq = true; + qp->has_rq = true; + qp->lockfree = false; + + list_node_init(&qp->cq_poll_sq); + list_node_init(&qp->cq_poll_rq); + list_node_init(&qp->cq_flush_sq); + list_node_init(&qp->cq_flush_rq); + + pthread_spin_init(&qp->sq.lock, PTHREAD_PROCESS_PRIVATE); + pthread_spin_init(&qp->rq.lock, PTHREAD_PROCESS_PRIVATE); + + qp->sq.cmb = pd->sq_cmb; + qp->rq.cmb = pd->rq_cmb; + + rc = ionic_qp_sq_init(ctx, qp, pd, ex->cap.max_send_wr, + ex->cap.max_send_sge, ex->cap.max_inline_data); + if (rc) + goto err_sq; + + rc = ionic_qp_rq_init(ctx, qp, pd, ex->cap.max_recv_wr, + ex->cap.max_recv_sge); + if (rc) + goto err_rq; + + req.sq.addr = (uintptr_t)qp->sq.queue.ptr; + req.sq.size = qp->sq.queue.size; + req.sq.mask = qp->sq.queue.mask; + req.sq.depth_log2 = qp->sq.queue.depth_log2; + req.sq.stride_log2 = qp->sq.queue.stride_log2; + req.sq_cmb = qp->sq.cmb; + + req.rq.addr = (uintptr_t)qp->rq.queue.ptr; + req.rq.size = qp->rq.queue.size; + req.rq.mask = qp->rq.queue.mask; + req.rq.depth_log2 = qp->rq.queue.depth_log2; + req.rq.stride_log2 = qp->rq.queue.stride_log2; + req.rq_cmb = qp->rq.cmb; + + req.sq_spec = qp->sq.spec; + req.rq_spec = qp->rq.spec; + + req.udma_mask = pd->udma_mask; + + rc = ibv_cmd_create_qp_ex2(ibctx, &qp->vqp, ex, + &req.ibv_cmd, + sizeof(req), + &resp.ibv_resp, + sizeof(resp)); + if (rc) + goto err_cmd; + + qp->qpid = resp.qpid; + qp->udma_idx = resp.udma_idx; + + ionic_queue_dbell_init(&qp->sq.queue, qp->qpid); + ionic_queue_dbell_init(&qp->rq.queue, qp->qpid); + + qp->sq.cmb = resp.sq_cmb; + qp->rq.cmb = resp.rq_cmb; + + rc = ionic_set_cmb(ctx, &qp->sq.queue, qp->sq.cmb, resp.sq_cmb_offset, + ctx->sq_qtype, &qp->sq.cmb_ptr); + if (rc) + goto err_cmb; + + rc = ionic_set_cmb(ctx, &qp->rq.queue, qp->rq.cmb, resp.rq_cmb_offset, + ctx->rq_qtype, &qp->rq.cmb_ptr); + if (rc) + goto err_cmb; + + pthread_mutex_lock(&ctx->mut); + ionic_tbl_alloc_node(&ctx->qp_tbl); + ionic_tbl_insert(&ctx->qp_tbl, qp, qp->qpid); + pthread_mutex_unlock(&ctx->mut); + + if (qp->has_sq) { + cq = to_ionic_vcq_cq(qp->vqp.qp.send_cq, qp->udma_idx); + ionic_cq_spin_lock(cq); + ionic_cq_spin_unlock(cq); + + ex->cap.max_send_wr = qp->sq.queue.mask; + ex->cap.max_send_sge = + ionic_v1_send_wqe_max_sge(qp->sq.queue.stride_log2, qp->sq.spec, + qp->sq.cmb & IONIC_CMB_EXPDB); + ex->cap.max_inline_data = + ionic_v1_send_wqe_max_data(qp->sq.queue.stride_log2, + qp->sq.cmb & IONIC_CMB_EXPDB); + } + + if (qp->has_rq) { + cq = to_ionic_vcq_cq(qp->vqp.qp.recv_cq, qp->udma_idx); + ionic_cq_spin_lock(cq); + ionic_cq_spin_unlock(cq); + + ex->cap.max_recv_wr = qp->rq.queue.mask; + ex->cap.max_recv_sge = + ionic_v1_recv_wqe_max_sge(qp->rq.queue.stride_log2, qp->rq.spec, + qp->rq.cmb & IONIC_CMB_EXPDB); + } + + return &qp->vqp.qp; + +err_cmb: + ibv_cmd_destroy_qp(&qp->vqp.qp); + ionic_unmap(qp->sq.cmb_ptr, qp->sq.queue.size); + ionic_unmap(qp->rq.cmb_ptr, qp->rq.queue.size); +err_cmd: + ionic_qp_rq_destroy(qp); +err_rq: + ionic_qp_sq_destroy(qp); +err_sq: + pthread_spin_destroy(&qp->rq.lock); + pthread_spin_destroy(&qp->sq.lock); + free(qp); +err_qp: + errno = rc; + return NULL; +} + +static void ionic_flush_qp(struct ionic_ctx *ctx, struct ionic_qp *qp) +{ + struct ionic_cq *cq; + + if (qp->vqp.qp.send_cq) { + cq = to_ionic_vcq_cq(qp->vqp.qp.send_cq, qp->udma_idx); + + /* Hold the CQ lock and QP sq_lock while setting up flush */ + ionic_cq_spin_lock(cq); + ionic_sq_spin_lock(qp); + qp->sq.flush = true; + + if (!ionic_queue_empty(&qp->sq.queue)) { + cq->flush = true; + list_del(&qp->cq_flush_sq); + list_add_tail(&cq->flush_sq, &qp->cq_flush_sq); + } + + ionic_sq_spin_unlock(qp); + ionic_cq_spin_unlock(cq); + } + + if (qp->vqp.qp.recv_cq) { + cq = to_ionic_vcq_cq(qp->vqp.qp.recv_cq, qp->udma_idx); + + /* Hold the CQ lock and QP rq_lock while setting up flush */ + ionic_cq_spin_lock(cq); + ionic_rq_spin_lock(qp); + qp->rq.flush = true; + + if (!ionic_queue_empty(&qp->rq.queue)) { + cq->flush = true; + list_del(&qp->cq_flush_rq); + list_add_tail(&cq->flush_rq, &qp->cq_flush_rq); + } + + ionic_rq_spin_unlock(qp); + ionic_cq_spin_unlock(cq); + } +} + +static void ionic_reset_qp(struct ionic_ctx *ctx, struct ionic_qp *qp) +{ + struct ionic_cq *cq; + int i; + + if (qp->vqp.qp.send_cq) { + cq = to_ionic_vcq_cq(qp->vqp.qp.send_cq, qp->udma_idx); + ionic_cq_spin_lock(cq); + ionic_clean_cq(cq, qp->qpid); + ionic_cq_spin_unlock(cq); + } + + if (qp->vqp.qp.recv_cq) { + cq = to_ionic_vcq_cq(qp->vqp.qp.recv_cq, qp->udma_idx); + ionic_cq_spin_lock(cq); + ionic_clean_cq(cq, qp->qpid); + ionic_cq_spin_unlock(cq); + } + + if (qp->has_sq) { + ionic_sq_spin_lock(qp); + qp->sq.flush = false; + qp->sq.flush_rcvd = false; + qp->sq.msn_prod = 0; + qp->sq.msn_cons = 0; + qp->sq.cmb_prod = 0; + qp->sq.old_prod = 0; + qp->sq.queue.prod = 0; + qp->sq.queue.cons = 0; + ionic_sq_spin_unlock(qp); + } + + if (qp->has_rq) { + ionic_rq_spin_lock(qp); + qp->rq.flush = false; + qp->rq.queue.prod = 0; + qp->rq.queue.cons = 0; + qp->rq.cmb_prod = 0; + qp->rq.old_prod = 0; + for (i = 0; i < qp->rq.queue.mask; ++i) + qp->rq.meta[i].next = &qp->rq.meta[i + 1]; + qp->rq.meta[i].next = IONIC_META_LAST; + qp->rq.meta_head = &qp->rq.meta[0]; + ionic_rq_spin_unlock(qp); + } +} + +static int ionic_modify_qp(struct ibv_qp *ibqp, + struct ibv_qp_attr *attr, + int attr_mask) +{ + struct ionic_ctx *ctx = to_ionic_ctx(ibqp->context); + struct ionic_qp *qp = to_ionic_qp(ibqp); + struct ibv_modify_qp cmd = {}; + int rc; + + if (!attr_mask) + return 0; + + rc = ibv_cmd_modify_qp(ibqp, attr, attr_mask, &cmd, sizeof(cmd)); + if (rc) + goto err_cmd; + + if (attr_mask & IBV_QP_STATE) { + if (attr->qp_state == IBV_QPS_ERR) + ionic_flush_qp(ctx, qp); + else if (attr->qp_state == IBV_QPS_RESET) + ionic_reset_qp(ctx, qp); + } + +err_cmd: + if (attr_mask & IBV_QP_STATE) + verbs_debug(&ctx->vctx, "modify qp %u state %u -> %u rc %d", + qp->qpid, qp->vqp.qp.state, attr->qp_state, rc); + + return rc; +} + +static int ionic_query_qp(struct ibv_qp *ibqp, + struct ibv_qp_attr *attr, + int attr_mask, + struct ibv_qp_init_attr *init_attr) +{ + struct ionic_qp *qp = to_ionic_qp(ibqp); + struct ionic_ctx *ctx = to_ionic_ctx(ibqp->context); + struct ibv_query_qp cmd; + int rc; + + rc = ibv_cmd_query_qp(ibqp, attr, attr_mask, init_attr, + &cmd, sizeof(cmd)); + + if (qp->has_sq) { + init_attr->cap.max_send_wr = qp->sq.queue.mask; + init_attr->cap.max_send_sge = + ionic_v1_send_wqe_max_sge(qp->sq.queue.stride_log2, qp->sq.spec, + qp->sq.cmb & IONIC_CMB_EXPDB); + init_attr->cap.max_inline_data = + ionic_v1_send_wqe_max_data(qp->sq.queue.stride_log2, + qp->sq.cmb & IONIC_CMB_EXPDB); + } + + if (qp->has_rq) { + init_attr->cap.max_recv_wr = qp->rq.queue.mask; + init_attr->cap.max_recv_sge = + ionic_v1_send_wqe_max_sge(qp->rq.queue.stride_log2, qp->rq.spec, + qp->rq.cmb & IONIC_CMB_EXPDB); + } + + attr->cap = init_attr->cap; + + verbs_debug(&ctx->vctx, "query qp %u attr_state %u rc %d", + qp->qpid, attr->qp_state, rc); + + return rc; +} + +static int ionic_destroy_qp(struct ibv_qp *ibqp) +{ + struct ionic_ctx *ctx = to_ionic_ctx(ibqp->context); + struct ionic_qp *qp = to_ionic_qp(ibqp); + struct ionic_cq *cq; + int rc; + + rc = ibv_cmd_destroy_qp(ibqp); + if (rc) + return rc; + + pthread_mutex_lock(&ctx->mut); + ionic_tbl_free_node(&ctx->qp_tbl); + ionic_tbl_delete(&ctx->qp_tbl, qp->qpid); + pthread_mutex_unlock(&ctx->mut); + + if (qp->vqp.qp.send_cq) { + cq = to_ionic_vcq_cq(qp->vqp.qp.send_cq, qp->udma_idx); + ionic_cq_spin_lock(cq); + ionic_clean_cq(cq, qp->qpid); + list_del(&qp->cq_poll_sq); + list_del(&qp->cq_flush_sq); + ionic_cq_spin_unlock(cq); + } + + if (qp->vqp.qp.recv_cq) { + cq = to_ionic_vcq_cq(qp->vqp.qp.recv_cq, qp->udma_idx); + ionic_cq_spin_lock(cq); + ionic_clean_cq(cq, qp->qpid); + list_del(&qp->cq_poll_rq); + list_del(&qp->cq_flush_rq); + ionic_cq_spin_unlock(cq); + } + + ionic_unmap(qp->sq.cmb_ptr, qp->sq.queue.size); + ionic_unmap(qp->rq.cmb_ptr, qp->rq.queue.size); + + pthread_spin_destroy(&qp->rq.lock); + pthread_spin_destroy(&qp->sq.lock); + ionic_qp_rq_destroy(qp); + ionic_qp_sq_destroy(qp); + free(qp); + + return 0; +} + +static int64_t ionic_prep_inline(void *data, uint32_t max_data, + struct ibv_sge *ibv_sgl, int num_sge) +{ + int64_t len = 0, sg_len; + int sg_i; + + for (sg_i = 0; sg_i < num_sge; ++sg_i) { + sg_len = ibv_sgl[sg_i].length; + + /* greater than max inline data is invalid */ + if (unlikely(len + sg_len > max_data)) + return -EINVAL; + + memcpy(data + len, (void *)(uintptr_t)ibv_sgl[sg_i].addr, sg_len); + + len += sg_len; + } + + return len; +} + +static int64_t ionic_v1_prep_pld(struct ionic_v1_wqe *wqe, + union ionic_v1_pld *pld, + int spec, uint32_t max_sge, + struct ibv_sge *ibv_sgl, + int num_sge) +{ + static const int64_t bit_31 = 1l << 31; + int64_t len = 0, sg_len; + struct ionic_sge *sgl; + __be32 *spec32 = NULL; + __be16 *spec16 = NULL; + int sg_i = 0; + + if (unlikely(num_sge < 0 || (uint32_t)num_sge > max_sge)) + return -EINVAL; + + if (spec && num_sge > IONIC_V1_SPEC_FIRST_SGE) { + sg_i = IONIC_V1_SPEC_FIRST_SGE; + + if (num_sge > 8) { + wqe->base.flags |= htobe16(IONIC_V1_FLAG_SPEC16); + spec16 = pld->spec16; + } else { + wqe->base.flags |= htobe16(IONIC_V1_FLAG_SPEC32); + spec32 = pld->spec32; + } + } + + sgl = &pld->sgl[sg_i]; + + for (sg_i = 0; sg_i < num_sge; ++sg_i) { + sg_len = ibv_sgl[sg_i].length; + + /* greater than 2GB data is invalid */ + if (unlikely(len + sg_len > bit_31)) + return -EINVAL; + + sgl[sg_i].va = htobe64(ibv_sgl[sg_i].addr); + sgl[sg_i].len = htobe32(sg_len); + sgl[sg_i].lkey = htobe32(ibv_sgl[sg_i].lkey); + + if (spec32) { + spec32[sg_i] = sgl[sg_i].len; + } else if (spec16) { + if (unlikely(sg_len > UINT16_MAX)) + return -EINVAL; + spec16[sg_i] = htobe16(sg_len); + } + + len += sg_len; + } + + return len; +} + +static void ionic_v1_prep_base(struct ionic_qp *qp, + struct ibv_send_wr *wr, + struct ionic_sq_meta *meta, + struct ionic_v1_wqe *wqe) +{ + struct ionic_ctx *ctx = to_ionic_ctx(qp->vqp.qp.context); + + meta->wrid = wr->wr_id; + meta->ibsts = IBV_WC_SUCCESS; + meta->signal = false; + meta->local_comp = false; + + wqe->base.wqe_id = qp->sq.queue.prod; + if (qp->sq.color) + wqe->base.flags |= htobe16(IONIC_V1_FLAG_COLOR); + + if (wr->send_flags & IBV_SEND_FENCE) + wqe->base.flags |= htobe16(IONIC_V1_FLAG_FENCE); + + if (wr->send_flags & IBV_SEND_SOLICITED) + wqe->base.flags |= htobe16(IONIC_V1_FLAG_SOL); + + if (wr->send_flags & IBV_SEND_SIGNALED) { + wqe->base.flags |= htobe16(IONIC_V1_FLAG_SIG); + meta->signal = true; + } + + meta->seq = qp->sq.msn_prod; + meta->remote = qp->vqp.qp.qp_type != IBV_QPT_UD && + !ionic_ibop_is_local(wr->opcode); + + if (meta->remote) { + qp->sq.msn_idx[meta->seq] = qp->sq.queue.prod; + qp->sq.msn_prod = ionic_queue_next(&qp->sq.queue, qp->sq.msn_prod); + } + + verbs_debug(&ctx->vctx, "post send %u prod %u", + qp->qpid, qp->sq.queue.prod); + ionic_dbg_xdump(ctx, "wqe", wqe, 1u << qp->sq.queue.stride_log2); + + ionic_queue_produce(&qp->sq.queue); + qp->sq.color = ionic_color_wrap(qp->sq.queue.prod, qp->sq.color); +} + +static int ionic_v1_prep_common(struct ionic_qp *qp, + struct ibv_send_wr *wr, + struct ionic_sq_meta *meta, + struct ionic_v1_wqe *wqe) +{ + int64_t signed_len; + uint32_t mval; + + if (wr->send_flags & IBV_SEND_INLINE) { + wqe->base.num_sge_key = 0; + wqe->base.flags |= htobe16(IONIC_V1_FLAG_INL); + mval = ionic_v1_send_wqe_max_data(qp->sq.queue.stride_log2, + qp->sq.cmb & IONIC_CMB_EXPDB); + signed_len = ionic_prep_inline(wqe->common.pld.data, mval, + wr->sg_list, wr->num_sge); + } else { + wqe->base.num_sge_key = wr->num_sge; + mval = ionic_v1_send_wqe_max_sge(qp->sq.queue.stride_log2, qp->sq.spec, + qp->sq.cmb & IONIC_CMB_EXPDB); + signed_len = ionic_v1_prep_pld(wqe, &wqe->common.pld, + qp->sq.spec, mval, + wr->sg_list, wr->num_sge); + } + + if (unlikely(signed_len < 0)) + return -signed_len; + + meta->len = signed_len; + wqe->common.length = htobe32(signed_len); + + ionic_v1_prep_base(qp, wr, meta, wqe); + + return 0; +} + +static void ionic_prep_sq_wqe(struct ionic_qp *qp, void *wqe) +{ + uint32_t wqe_sz = 1u << qp->sq.queue.stride_log2; + + if (qp->sq.cmb & IONIC_CMB_EXPDB) + memset(wqe, 0, wqe_sz - IONIC_EXP_DBELL_SZ); + else + memset(wqe, 0, wqe_sz); +} + +static void ionic_prep_rq_wqe(struct ionic_qp *qp, void *wqe) +{ + uint32_t wqe_sz = 1u << qp->rq.queue.stride_log2; + + if (qp->rq.cmb & IONIC_CMB_EXPDB) + memset(wqe, 0, wqe_sz - IONIC_EXP_DBELL_SZ); + else + memset(wqe, 0, wqe_sz); +} + +static int ionic_v1_prep_send(struct ionic_qp *qp, + struct ibv_send_wr *wr) +{ + struct ionic_ctx *ctx = to_ionic_ctx(qp->vqp.qp.context); + struct ionic_sq_meta *meta; + struct ionic_v1_wqe *wqe; + + meta = &qp->sq.meta[qp->sq.queue.prod]; + wqe = ionic_queue_at_prod(&qp->sq.queue); + + ionic_prep_sq_wqe(qp, wqe); + + meta->ibop = IBV_WC_SEND; + + switch (wr->opcode) { + case IBV_WR_SEND: + wqe->base.op = IONIC_OP(ctx->version, SEND); + break; + case IBV_WR_SEND_WITH_IMM: + wqe->base.op = IONIC_OP(ctx->version, SEND_IMM); + wqe->base.imm_data_key = wr->imm_data; + break; + case IBV_WR_SEND_WITH_INV: + wqe->base.op = IONIC_OP(ctx->version, SEND_INV); + wqe->base.imm_data_key = htobe32(wr->invalidate_rkey); + break; + default: + return EINVAL; + } + + return ionic_v1_prep_common(qp, wr, meta, wqe); +} + +static int ionic_v1_prep_send_ud(struct ionic_qp *qp, struct ibv_send_wr *wr) +{ + struct ionic_ctx *ctx = to_ionic_ctx(qp->vqp.qp.context); + struct ionic_sq_meta *meta; + struct ionic_v1_wqe *wqe; + struct ionic_ah *ah; + + if (unlikely(!wr->wr.ud.ah)) + return EINVAL; + + ah = to_ionic_ah(wr->wr.ud.ah); + + meta = &qp->sq.meta[qp->sq.queue.prod]; + wqe = ionic_queue_at_prod(&qp->sq.queue); + + ionic_prep_sq_wqe(qp, wqe); + + wqe->common.send.ah_id = htobe32(ah->ahid); + wqe->common.send.dest_qpn = htobe32(wr->wr.ud.remote_qpn); + wqe->common.send.dest_qkey = htobe32(wr->wr.ud.remote_qkey); + + meta->ibop = IBV_WC_SEND; + + switch (wr->opcode) { + case IBV_WR_SEND: + wqe->base.op = IONIC_OP(ctx->version, SEND); + break; + case IBV_WR_SEND_WITH_IMM: + wqe->base.op = IONIC_OP(ctx->version, SEND_IMM); + wqe->base.imm_data_key = wr->imm_data; + break; + default: + return EINVAL; + } + + return ionic_v1_prep_common(qp, wr, meta, wqe); +} + +static int ionic_v1_prep_rdma(struct ionic_qp *qp, + struct ibv_send_wr *wr) +{ + struct ionic_ctx *ctx = to_ionic_ctx(qp->vqp.qp.context); + struct ionic_sq_meta *meta; + struct ionic_v1_wqe *wqe; + + meta = &qp->sq.meta[qp->sq.queue.prod]; + wqe = ionic_queue_at_prod(&qp->sq.queue); + + ionic_prep_sq_wqe(qp, wqe); + + meta->ibop = IBV_WC_RDMA_WRITE; + + switch (wr->opcode) { + case IBV_WR_RDMA_READ: + if (wr->send_flags & (IBV_SEND_SOLICITED | IBV_SEND_INLINE)) + return EINVAL; + meta->ibop = IBV_WC_RDMA_READ; + wqe->base.op = IONIC_OP(ctx->version, RDMA_READ); + break; + case IBV_WR_RDMA_WRITE: + if (wr->send_flags & IBV_SEND_SOLICITED) + return EINVAL; + wqe->base.op = IONIC_OP(ctx->version, RDMA_WRITE); + break; + case IBV_WR_RDMA_WRITE_WITH_IMM: + wqe->base.op = IONIC_OP(ctx->version, RDMA_WRITE_IMM); + wqe->base.imm_data_key = wr->imm_data; + break; + default: + return EINVAL; + } + + wqe->common.rdma.remote_va_high = + htobe32(wr->wr.rdma.remote_addr >> 32); + wqe->common.rdma.remote_va_low = htobe32(wr->wr.rdma.remote_addr); + wqe->common.rdma.remote_rkey = htobe32(wr->wr.rdma.rkey); + + return ionic_v1_prep_common(qp, wr, meta, wqe); +} + +static int ionic_v1_prep_atomic(struct ionic_qp *qp, + struct ibv_send_wr *wr) +{ + struct ionic_ctx *ctx = to_ionic_ctx(qp->vqp.qp.context); + struct ionic_sq_meta *meta; + struct ionic_v1_wqe *wqe; + + if (wr->num_sge != 1 || wr->sg_list[0].length != 8) + return EINVAL; + + if (wr->send_flags & (IBV_SEND_SOLICITED | IBV_SEND_INLINE)) + return EINVAL; + + meta = &qp->sq.meta[qp->sq.queue.prod]; + wqe = ionic_queue_at_prod(&qp->sq.queue); + + ionic_prep_sq_wqe(qp, wqe); + + switch (wr->opcode) { + case IBV_WR_ATOMIC_CMP_AND_SWP: + meta->ibop = IBV_WC_COMP_SWAP; + wqe->base.op = IONIC_OP(ctx->version, ATOMIC_CS); + wqe->atomic.swap_add_high = htobe32(wr->wr.atomic.swap >> 32); + wqe->atomic.swap_add_low = htobe32(wr->wr.atomic.swap); + wqe->atomic.compare_high = + htobe32(wr->wr.atomic.compare_add >> 32); + wqe->atomic.compare_low = htobe32(wr->wr.atomic.compare_add); + break; + case IBV_WR_ATOMIC_FETCH_AND_ADD: + meta->ibop = IBV_WC_FETCH_ADD; + wqe->base.op = IONIC_OP(ctx->version, ATOMIC_FA); + wqe->atomic.swap_add_high = + htobe32(wr->wr.atomic.compare_add >> 32); + wqe->atomic.swap_add_low = htobe32(wr->wr.atomic.compare_add); + break; + default: + return EINVAL; + } + + wqe->atomic.remote_va_high = htobe32(wr->wr.atomic.remote_addr >> 32); + wqe->atomic.remote_va_low = htobe32(wr->wr.atomic.remote_addr); + wqe->atomic.remote_rkey = htobe32(wr->wr.atomic.rkey); + + wqe->base.num_sge_key = 1; + + /* + * The fields above are common to atomic and atomic_v2. Deal now with + * the fields that differ. + */ + if (likely(ctx->version >= 2)) { + wqe->atomic_v2.local_va = htobe64(wr->sg_list[0].addr); + wqe->atomic_v2.lkey = htobe32(wr->sg_list[0].lkey); + } else { + wqe->atomic.sge.va = htobe64(wr->sg_list[0].addr); + wqe->atomic.sge.len = htobe32(8); + wqe->atomic.sge.lkey = htobe32(wr->sg_list[0].lkey); + } + + ionic_v1_prep_base(qp, wr, meta, wqe); + + return 0; +} + +static int ionic_v1_prep_inv(struct ionic_qp *qp, struct ibv_send_wr *wr) +{ + struct ionic_ctx *ctx = to_ionic_ctx(qp->vqp.qp.context); + struct ionic_sq_meta *meta; + struct ionic_v1_wqe *wqe; + + if (wr->send_flags & (IBV_SEND_SOLICITED | IBV_SEND_INLINE)) + return EINVAL; + + meta = &qp->sq.meta[qp->sq.queue.prod]; + meta->ibop = IBV_WC_LOCAL_INV; + wqe = ionic_queue_at_prod(&qp->sq.queue); + + ionic_prep_sq_wqe(qp, wqe); + + wqe->base.op = IONIC_OP(ctx->version, LOCAL_INV); + wqe->base.imm_data_key = htobe32(wr->invalidate_rkey); + + ionic_v1_prep_base(qp, wr, meta, wqe); + + return 0; +} + +static int ionic_v1_prep_bind(struct ionic_qp *qp, + struct ibv_send_wr *wr, + bool send_path) +{ + struct ionic_ctx *ctx = to_ionic_ctx(qp->vqp.qp.context); + struct ionic_sq_meta *meta; + struct ionic_v1_wqe *wqe; + int flags; + + if (wr->send_flags & (IBV_SEND_SOLICITED | IBV_SEND_INLINE)) + return EINVAL; + + /* type 1 must use bind_mw; type 2 must use post_send */ + if (send_path == (wr->bind_mw.mw->type == IBV_MW_TYPE_1)) + return EINVAL; + + /* only type 1 can unbind with zero length */ + if (!wr->bind_mw.bind_info.length && + wr->bind_mw.mw->type != IBV_MW_TYPE_1) + return EINVAL; + + meta = &qp->sq.meta[qp->sq.queue.prod]; + meta->ibop = IBV_WC_BIND_MW; + wqe = ionic_queue_at_prod(&qp->sq.queue); + + ionic_prep_sq_wqe(qp, wqe); + + flags = to_ionic_mr_flags(wr->bind_mw.bind_info.mw_access_flags); + + if (wr->bind_mw.mw->type == IBV_MW_TYPE_1) + flags |= IONIC_MRF_MW_1; + else + flags |= IONIC_MRF_MW_2; + + wqe->base.op = IONIC_OP(ctx->version, BIND_MW); + wqe->base.num_sge_key = wr->bind_mw.rkey; + wqe->base.imm_data_key = htobe32(wr->bind_mw.mw->rkey); + wqe->bind_mw.va = htobe64(wr->bind_mw.bind_info.addr); + wqe->bind_mw.length = htobe64(wr->bind_mw.bind_info.length); + wqe->bind_mw.lkey = htobe32(wr->bind_mw.bind_info.mr->lkey); + wqe->bind_mw.flags = htobe16(flags); + + ionic_v1_prep_base(qp, wr, meta, wqe); + + return 0; +} + +static int ionic_v1_prep_one_rc(struct ionic_qp *qp, + struct ibv_send_wr *wr, + bool send_path) +{ + struct ionic_ctx *ctx = to_ionic_ctx(qp->vqp.qp.context); + int rc = 0; + + switch (wr->opcode) { + case IBV_WR_SEND: + case IBV_WR_SEND_WITH_IMM: + case IBV_WR_SEND_WITH_INV: + rc = ionic_v1_prep_send(qp, wr); + break; + case IBV_WR_RDMA_READ: + case IBV_WR_RDMA_WRITE: + case IBV_WR_RDMA_WRITE_WITH_IMM: + rc = ionic_v1_prep_rdma(qp, wr); + break; + case IBV_WR_ATOMIC_CMP_AND_SWP: + case IBV_WR_ATOMIC_FETCH_AND_ADD: + rc = ionic_v1_prep_atomic(qp, wr); + break; + case IBV_WR_LOCAL_INV: + rc = ionic_v1_prep_inv(qp, wr); + break; + case IBV_WR_BIND_MW: + rc = ionic_v1_prep_bind(qp, wr, send_path); + break; + default: + verbs_warn(&ctx->vctx, "invalid opcode %d", wr->opcode); + rc = EINVAL; + } + + return rc; +} + +static int ionic_v1_prep_one_ud(struct ionic_qp *qp, + struct ibv_send_wr *wr) +{ + struct ionic_ctx *ctx = to_ionic_ctx(qp->vqp.qp.context); + int rc = 0; + + switch (wr->opcode) { + case IBV_WR_SEND: + case IBV_WR_SEND_WITH_IMM: + rc = ionic_v1_prep_send_ud(qp, wr); + break; + default: + verbs_warn(&ctx->vctx, "invalid opcode %d", wr->opcode); + rc = EINVAL; + } + + return rc; +} + +static void ionic_post_cmb_common(struct ionic_ctx *ctx, + struct ionic_queue *queue, + uint8_t cmb, + void *cmb_ptr, + uint16_t *cmb_prod, + uint8_t qtype, + uint32_t qpid) +{ + void *cmb_dst_ptr; + void *wqe_src_ptr; + uint32_t stride; + uint16_t pos, end; + uint8_t stride_log2, cmb_wc; + + stride_log2 = queue->stride_log2; + + pos = *cmb_prod; + end = queue->prod; + + cmb_wc = !(cmb & IONIC_CMB_UC); + + if (cmb & IONIC_CMB_EXPDB) { + /* Express doorbell mode: copy each WQE individually with barriers */ + while (pos != end) { + cmb_dst_ptr = cmb_ptr + ((size_t)pos << stride_log2); + wqe_src_ptr = ionic_queue_at(queue, pos); + + stride = 1u << stride_log2; + do { + udma_to_device_barrier(); + /* only fence before last 64B of each + * WC wqe, no need to mmio_wc_start() + */ + if (cmb_wc && stride <= 64) + mmio_flush_writes(); + ionic_mmio_memcpy_x64_64(cmb_dst_ptr, wqe_src_ptr); + + stride -= 64; + cmb_dst_ptr += 64; + wqe_src_ptr += 64; + } while (stride); + + pos = ionic_queue_next(queue, pos); + } + } else { + /* Regular doorbell mode: bulk copy with wrap-around handling */ + if (pos > end) { + /* Handle wrap-around case: copy from pos to end of ring */ + cmb_dst_ptr = cmb_ptr + ((size_t)pos << stride_log2); + wqe_src_ptr = ionic_queue_at(queue, pos); + stride = (uint32_t)(queue->mask - pos + 1) << stride_log2; + + ionic_mmio_memcpy_x64(cmb_dst_ptr, wqe_src_ptr, stride); + + pos = 0; + } + + if (pos < end) { + /* Copy from pos to end */ + cmb_dst_ptr = cmb_ptr + ((size_t)pos << stride_log2); + wqe_src_ptr = ionic_queue_at(queue, pos); + stride = (uint32_t)(end - pos) << stride_log2; + + ionic_mmio_memcpy_x64(cmb_dst_ptr, wqe_src_ptr, stride); + + pos = end; + } + + if (cmb_wc) + mmio_flush_writes(); + + /* Ring doorbell */ + verbs_debug(&ctx->vctx, "dbell qp %u qtype %d val %" PRIx64, + qpid, qtype, queue->dbell | pos); + ionic_dbell_ring(&ctx->dbpage[qtype], queue->dbell | pos); + } + + *cmb_prod = end; +} + +static void ionic_post_send_cmb(struct ionic_ctx *ctx, struct ionic_qp *qp) +{ + ionic_post_cmb_common(ctx, + &qp->sq.queue, + qp->sq.cmb, + qp->sq.cmb_ptr, + &qp->sq.cmb_prod, + ctx->sq_qtype, + qp->qpid); +} + +static void ionic_post_recv_cmb(struct ionic_ctx *ctx, struct ionic_qp *qp) +{ + ionic_post_cmb_common(ctx, + &qp->rq.queue, + qp->rq.cmb, + qp->rq.cmb_ptr, + &qp->rq.cmb_prod, + ctx->rq_qtype, + qp->qpid); +} + +static int ionic_post_send_common(struct ionic_ctx *ctx, + struct ionic_cq *cq, + struct ionic_qp *qp, + struct ibv_send_wr *wr, + struct ibv_send_wr **bad, + bool send_path) +{ + int spend, rc = 0; + uint16_t old_prod; + + if (unlikely(!bad)) + return EINVAL; + + if (unlikely(!qp->has_sq)) { + *bad = wr; + return EINVAL; + } + + if (unlikely(qp->vqp.qp.state < IBV_QPS_RTS)) { + *bad = wr; + return EINVAL; + } + + ionic_sq_spin_lock(qp); + + old_prod = qp->sq.queue.prod; + + while (wr) { + if (ionic_queue_full(&qp->sq.queue)) { + verbs_info(&ctx->vctx, + "send queue full cons %u prod %u", + qp->sq.queue.cons, qp->sq.queue.prod); + rc = ENOMEM; + goto out; + } + + if (qp->vqp.qp.qp_type == IBV_QPT_UD) + rc = ionic_v1_prep_one_ud(qp, wr); + else + rc = ionic_v1_prep_one_rc(qp, wr, send_path); + if (rc) + goto out; + + wr = wr->next; + } + +out: + old_prod = (qp->sq.queue.prod - old_prod) & qp->sq.queue.mask; + + if (ionic_cq_spin_trylock(cq)) { + ionic_sq_spin_unlock(qp); + ionic_cq_spin_lock(cq); + ionic_sq_spin_lock(qp); + } + + if (likely(qp->sq.queue.prod != qp->sq.old_prod)) { + /* ring cq doorbell just in time */ + spend = (qp->sq.queue.prod - qp->sq.old_prod) & qp->sq.queue.mask; + ionic_reserve_cq(ctx, cq, spend); + + qp->sq.old_prod = qp->sq.queue.prod; + + if (qp->sq.cmb_ptr) { + ionic_post_send_cmb(ctx, qp); + } else { + udma_to_device_barrier(); + verbs_debug(&ctx->vctx, "dbell qp %u sq val %" PRIx64, + qp->qpid, ionic_queue_dbell_val(&qp->sq.queue)); + ionic_dbell_ring(&ctx->dbpage[ctx->sq_qtype], + ionic_queue_dbell_val(&qp->sq.queue)); + } + } + + if (qp->sq.flush) { + cq->flush = true; + list_del(&qp->cq_flush_sq); + list_add_tail(&cq->flush_sq, &qp->cq_flush_sq); + } + + ionic_sq_spin_unlock(qp); + ionic_cq_spin_unlock(cq); + + *bad = wr; + return rc; +} + +static int ionic_v1_prep_recv(struct ionic_qp *qp, + struct ibv_recv_wr *wr) +{ + struct ionic_ctx *ctx = to_ionic_ctx(qp->vqp.qp.context); + struct ionic_rq_meta *meta; + struct ionic_v1_wqe *wqe; + int64_t signed_len; + uint32_t mval; + + wqe = ionic_queue_at_prod(&qp->rq.queue); + + /* if wqe is owned by device, caller can try posting again soon */ + if (wqe->base.flags & htobe16(IONIC_V1_FLAG_FENCE)) + return -EAGAIN; + + meta = qp->rq.meta_head; + if (unlikely(meta == IONIC_META_LAST) || + unlikely(meta == IONIC_META_POSTED)) + return -EIO; + + ionic_prep_rq_wqe(qp, wqe); + + mval = ionic_v1_recv_wqe_max_sge(qp->rq.queue.stride_log2, + qp->rq.spec, qp->rq.cmb & IONIC_CMB_EXPDB); + signed_len = ionic_v1_prep_pld(wqe, &wqe->recv.pld, + qp->rq.spec, mval, + wr->sg_list, wr->num_sge); + if (signed_len < 0) + return -signed_len; + + meta->wrid = wr->wr_id; + + wqe->base.wqe_id = qp->rq.queue.prod; + wqe->base.num_sge_key = wr->num_sge; + + qp->rq.meta_idx[qp->rq.queue.prod] = meta - qp->rq.meta; + + /* total length for recv goes in base imm_data_key */ + wqe->base.imm_data_key = htobe32(signed_len); + + verbs_debug(&ctx->vctx, "post recv %u prod %u", + qp->qpid, qp->rq.queue.prod); + ionic_dbg_xdump(ctx, "wqe", wqe, 1u << qp->rq.queue.stride_log2); + ionic_queue_produce(&qp->rq.queue); + + qp->rq.meta_head = meta->next; + meta->next = IONIC_META_POSTED; + + return 0; +} + +static int ionic_post_recv_common(struct ionic_ctx *ctx, + struct ionic_cq *cq, + struct ionic_qp *qp, + struct ibv_recv_wr *wr, + struct ibv_recv_wr **bad) +{ + int spend, rc = 0; + uint16_t old_prod; + + if (unlikely(!bad)) + return EINVAL; + + if (unlikely(!qp->has_rq)) { + *bad = wr; + return EINVAL; + } + + if (unlikely(qp->vqp.qp.state < IBV_QPS_INIT)) { + *bad = wr; + return EINVAL; + } + + ionic_rq_spin_lock(qp); + + old_prod = qp->rq.queue.prod; + + while (wr) { + if (ionic_queue_full(&qp->rq.queue)) { + verbs_info(&ctx->vctx, "recv queue full cons %u prod %u", + qp->rq.queue.cons, qp->rq.queue.prod); + rc = ENOMEM; + goto out; + } + + rc = ionic_v1_prep_recv(qp, wr); + if (rc) + goto out; + + wr = wr->next; + } + +out: + old_prod = (qp->rq.queue.prod - old_prod) & qp->rq.queue.mask; + + if (!cq) { + ionic_rq_spin_unlock(qp); + goto out_unlocked; + } + + if (ionic_cq_spin_trylock(cq)) { + ionic_rq_spin_unlock(qp); + ionic_cq_spin_lock(cq); + ionic_rq_spin_lock(qp); + } + + if (likely(qp->rq.queue.prod != qp->rq.old_prod)) { + /* ring cq doorbell just in time */ + spend = (qp->rq.queue.prod - qp->rq.old_prod) & qp->rq.queue.mask; + ionic_reserve_cq(ctx, cq, spend); + + qp->rq.old_prod = qp->rq.queue.prod; + + if (qp->rq.cmb_ptr) { + ionic_post_recv_cmb(ctx, qp); + } else { + udma_to_device_barrier(); + verbs_debug(&ctx->vctx, "dbell qp %u rq val %" PRIx64, + qp->qpid, + ionic_queue_dbell_val(&qp->rq.queue)); + ionic_dbell_ring(&ctx->dbpage[ctx->rq_qtype], + ionic_queue_dbell_val(&qp->rq.queue)); + } + } + + if (qp->rq.flush) { + cq->flush = true; + list_del(&qp->cq_flush_rq); + list_add_tail(&cq->flush_rq, &qp->cq_flush_rq); + } + + ionic_rq_spin_unlock(qp); + ionic_cq_spin_unlock(cq); + +out_unlocked: + *bad = wr; + + return rc; +} + +static int ionic_post_send(struct ibv_qp *ibqp, + struct ibv_send_wr *wr, + struct ibv_send_wr **bad) +{ + struct ionic_ctx *ctx = to_ionic_ctx(ibqp->context); + struct ionic_qp *qp = to_ionic_qp(ibqp); + struct ionic_cq *cq = to_ionic_vcq_cq(qp->vqp.qp.send_cq, qp->udma_idx); + + return ionic_post_send_common(ctx, cq, qp, wr, bad, true); +} + +static int ionic_post_recv(struct ibv_qp *ibqp, + struct ibv_recv_wr *wr, + struct ibv_recv_wr **bad) +{ + struct ionic_ctx *ctx = to_ionic_ctx(ibqp->context); + struct ionic_qp *qp = to_ionic_qp(ibqp); + struct ionic_cq *cq = to_ionic_vcq_cq(qp->vqp.qp.recv_cq, qp->udma_idx); + + return ionic_post_recv_common(ctx, cq, qp, wr, bad); +} + +static struct ibv_qp *ionic_create_qp(struct ibv_pd *ibpd, + struct ibv_qp_init_attr *attr) +{ + struct ibv_qp_init_attr_ex ex = { + .qp_context = attr->qp_context, + .send_cq = attr->send_cq, + .recv_cq = attr->recv_cq, + .srq = attr->srq, + .cap = attr->cap, + .qp_type = attr->qp_type, + .sq_sig_all = attr->sq_sig_all, + .comp_mask = IBV_QP_INIT_ATTR_PD, + .pd = ibpd, + }; + struct verbs_context *vctx; + struct ibv_qp *ibqp; + + vctx = container_of(ibpd->context, struct verbs_context, context); + ibqp = vctx->create_qp_ex(&vctx->context, &ex); + + attr->cap = ex.cap; + + return ibqp; +} + +static struct ibv_ah *ionic_create_ah(struct ibv_pd *ibpd, + struct ibv_ah_attr *attr) +{ + struct ibv_pd *root_ibpd = to_ionic_root_ibpd(ibpd); + struct uionic_ah_resp resp; + struct ionic_ah *ah; + int rc; + + ah = calloc(1, sizeof(*ah)); + if (!ah) { + rc = errno; + goto err_ah; + } + + rc = ibv_cmd_create_ah(root_ibpd, &ah->ibah, attr, + &resp.ibv_resp, sizeof(resp)); + if (rc) + goto err_cmd; + + ah->ahid = resp.ahid; + + return &ah->ibah; + +err_cmd: + free(ah); +err_ah: + errno = rc; + return NULL; +} + +static int ionic_destroy_ah(struct ibv_ah *ibah) +{ + struct ionic_ah *ah = to_ionic_ah(ibah); + int rc; + + rc = ibv_cmd_destroy_ah(ibah); + if (rc) + return rc; + + free(ah); + + return 0; +} + +static struct ibv_mw *ionic_alloc_mw(struct ibv_pd *ibpd, + enum ibv_mw_type type) +{ + struct ibv_pd *root_ibpd = to_ionic_root_ibpd(ibpd); + struct ib_uverbs_alloc_mw_resp resp; + struct ibv_alloc_mw cmd; + struct ibv_mw *ibmw; + int rc; + + ibmw = calloc(1, sizeof(*ibmw)); + if (!ibmw) { + rc = errno; + goto err_mw; + } + + rc = ibv_cmd_alloc_mw(root_ibpd, type, ibmw, + &cmd, sizeof(cmd), + &resp, sizeof(resp)); + if (rc) + goto err_cmd; + + return ibmw; + +err_cmd: + free(ibmw); +err_mw: + errno = rc; + return NULL; +} + +static int ionic_bind_mw(struct ibv_qp *ibqp, struct ibv_mw *ibmw, + struct ibv_mw_bind *bind) +{ + struct ionic_ctx *ctx = to_ionic_ctx(ibqp->context); + struct ionic_qp *qp = to_ionic_qp(ibqp); + struct ionic_cq *cq = to_ionic_vcq_cq(qp->vqp.qp.send_cq, qp->udma_idx); + struct ibv_send_wr *bad; + struct ibv_send_wr wr = { + .opcode = IBV_WR_BIND_MW, + .wr_id = bind->wr_id, + .send_flags = bind->send_flags, + .bind_mw = { + .mw = ibmw, + .rkey = ibmw->rkey, + .bind_info = bind->bind_info, + } + }; + int rc; + + if (bind->bind_info.length) + wr.bind_mw.rkey = ibv_inc_rkey(ibmw->rkey); + + rc = ionic_post_send_common(ctx, cq, qp, &wr, &bad, false); + if (!rc) + ibmw->rkey = wr.bind_mw.rkey; + + return rc; +} + +static int ionic_dealloc_mw(struct ibv_mw *ibmw) +{ + int rc; + + rc = ibv_cmd_dealloc_mw(ibmw); + if (rc) + return rc; + + free(ibmw); + + return 0; +} + +static void ionic_free_context(struct ibv_context *ibctx) +{ + struct ionic_ctx *ctx = to_ionic_ctx(ibctx); + + ionic_tbl_destroy(&ctx->qp_tbl); + + pthread_mutex_destroy(&ctx->mut); + + ionic_unmap(ctx->dbpage_page, 1u << ctx->pg_shift); + + verbs_uninit_context(&ctx->vctx); + + free(ctx); +} + +bool is_ionic_ctx(struct ibv_context *ibctx) +{ + /* whatever we do here must be safe with non-ionic ibctx. */ + struct verbs_context *vctx = verbs_get_ctx_op(ibctx, alloc_parent_domain); + + return vctx && vctx->alloc_parent_domain == ionic_alloc_parent_domain; +} + +static const struct verbs_context_ops ionic_ctx_ops = { + .query_device_ex = ionic_query_device_ex, + .query_port = ionic_query_port, + .alloc_parent_domain = ionic_alloc_parent_domain, + .alloc_pd = ionic_alloc_pd, + .dealloc_pd = ionic_dealloc_pd, + .reg_dmabuf_mr = ionic_reg_dmabuf_mr, + .reg_mr = ionic_reg_mr, + .dereg_mr = ionic_dereg_mr, + .create_cq = ionic_create_cq, + .create_cq_ex = ionic_create_cq_ex, + .poll_cq = ionic_poll_cq, + .req_notify_cq = ionic_req_notify_cq, + .destroy_cq = ionic_destroy_cq, + .create_qp = ionic_create_qp, + .create_qp_ex = ionic_create_qp_ex, + .query_qp = ionic_query_qp, + .modify_qp = ionic_modify_qp, + .destroy_qp = ionic_destroy_qp, + .post_send = ionic_post_send, + .post_recv = ionic_post_recv, + .create_ah = ionic_create_ah, + .destroy_ah = ionic_destroy_ah, + .alloc_mw = ionic_alloc_mw, + .bind_mw = ionic_bind_mw, + .dealloc_mw = ionic_dealloc_mw, + .free_context = ionic_free_context, +}; + +void ionic_verbs_set_ops(struct ionic_ctx *ctx) +{ + verbs_set_ops(&ctx->vctx, &ionic_ctx_ops); +} diff --git a/providers/ionic/libionic.map b/providers/ionic/libionic.map new file mode 100644 index 000000000..3efca32b9 --- /dev/null +++ b/providers/ionic/libionic.map @@ -0,0 +1,12 @@ +/* Export symbols should be added below according to + Documentation/versioning.md document. */ +IONIC_1.0 { + global: + ionic_dv_ctx_get_udma_count; + ionic_dv_ctx_get_udma_mask; + ionic_dv_pd_get_udma_mask; + ionic_dv_pd_set_udma_mask; + ionic_dv_pd_set_sqcmb; + ionic_dv_pd_set_rqcmb; + local: *; +}; diff --git a/redhat/rdma-core.spec b/redhat/rdma-core.spec index 83ba45cc7..063d1f0e5 100644 --- a/redhat/rdma-core.spec +++ b/redhat/rdma-core.spec @@ -160,6 +160,8 @@ Provides: libhfi1 = %{version}-%{release} Obsoletes: libhfi1 < %{version}-%{release} Provides: libhns = %{version}-%{release} Obsoletes: libhns < %{version}-%{release} +Provides: libionic = %{version}-%{release} +Obsoletes: libionic < %{version}-%{release} Provides: libipathverbs = %{version}-%{release} Obsoletes: libipathverbs < %{version}-%{release} Provides: libirdma = %{version}-%{release} @@ -191,6 +193,7 @@ Device-specific plug-in ibverbs userspace drivers are included: - liberdma: Alibaba Elastic RDMA (iWarp) Adapter - libhfi1: Intel Omni-Path HFI - libhns: HiSilicon Hip08+ SoC +- libionic: AMD Pensando Distributed Services Card (DSC) RDMA/RoCE Support - libipathverbs: QLogic InfiniPath HCA - libirdma: Intel Ethernet Connection RDMA - libmana: Microsoft Azure Network Adapter @@ -594,6 +597,7 @@ fi %{_libdir}/libhns.so.* %{_libdir}/libibverbs*.so.* %{_libdir}/libibverbs/*.so +%{_libdir}/libionic.so.* %{_libdir}/libmana.so.* %{_libdir}/libmlx5.so.* %{_libdir}/libmlx4.so.* diff --git a/suse/rdma-core.spec b/suse/rdma-core.spec index 99e9b3796..29aa55638 100644 --- a/suse/rdma-core.spec +++ b/suse/rdma-core.spec @@ -36,6 +36,7 @@ Group: Productivity/Networking/Other %define efa_so_major 1 %define hns_so_major 1 +%define ionic_so_major 1 %define verbs_so_major 1 %define rdmacm_so_major 1 %define umad_so_major 3 @@ -47,6 +48,7 @@ Group: Productivity/Networking/Other %define efa_lname libefa%{efa_so_major} %define hns_lname libhns%{hns_so_major} +%define ionic_lname libionic%{ionic_so_major} %define verbs_lname libibverbs%{verbs_so_major} %define rdmacm_lname librdmacm%{rdmacm_so_major} %define umad_lname libibumad%{umad_so_major} @@ -162,6 +164,7 @@ Requires: %{verbs_lname} = %{version}-%{release} %if 0%{?dma_coherent} Requires: %{efa_lname} = %{version}-%{release} Requires: %{hns_lname} = %{version}-%{release} +Requires: %{ionic_lname} = %{version}-%{release} Requires: %{mana_lname} = %{version}-%{release} Requires: %{mlx4_lname} = %{version}-%{release} Requires: %{mlx5_lname} = %{version}-%{release} @@ -204,6 +207,7 @@ Obsoletes: libcxgb4-rdmav2 < %{version}-%{release} Obsoletes: libefa-rdmav2 < %{version}-%{release} Obsoletes: libhfi1verbs-rdmav2 < %{version}-%{release} Obsoletes: libhns-rdmav2 < %{version}-%{release} +Obsoletes: libionic-rdmav2 < %{version}-%{release} Obsoletes: libipathverbs-rdmav2 < %{version}-%{release} Obsoletes: libmana-rdmav2 < %{version}-%{release} Obsoletes: libmlx4-rdmav2 < %{version}-%{release} @@ -214,6 +218,7 @@ Obsoletes: librxe-rdmav2 < %{version}-%{release} %if 0%{?dma_coherent} Requires: %{efa_lname} = %{version}-%{release} Requires: %{hns_lname} = %{version}-%{release} +Requires: %{ionic_lname} = %{version}-%{release} Requires: %{mana_lname} = %{version}-%{release} Requires: %{mlx4_lname} = %{version}-%{release} Requires: %{mlx5_lname} = %{version}-%{release} @@ -234,6 +239,7 @@ Device-specific plug-in ibverbs userspace drivers are included: - libefa: Amazon Elastic Fabric Adapter - libhfi1: Intel Omni-Path HFI - libhns: HiSilicon Hip08+ SoC +- libionic: AMD Pensando Distributed Services Card (DSC) RDMA/RoCE Support - libipathverbs: QLogic InfiniPath HCA - libirdma: Intel Ethernet Connection RDMA - libmana: Microsoft Azure Network Adapter @@ -268,6 +274,13 @@ Group: System/Libraries %description -n %hns_lname This package contains the hns runtime library. +%package -n %ionic_lname +Summary: IONIC runtime library +Group: System/Libraries + +%description -n %ionic_lname +This package contains the ionic runtime library. + %package -n %mana_lname Summary: MANA runtime library Group: System/Libraries @@ -525,6 +538,9 @@ rm -rf %{buildroot}/%{_sbindir}/srp_daemon.sh %post -n %hns_lname -p /sbin/ldconfig %postun -n %hns_lname -p /sbin/ldconfig +%post -n %ionic_lname -p /sbin/ldconfig +%postun -n %ionic_lname -p /sbin/ldconfig + %post -n %mana_lname -p /sbin/ldconfig %postun -n %mana_lname -p /sbin/ldconfig @@ -725,6 +741,9 @@ done %defattr(-,root,root) %{_libdir}/libhns*.so.* +%files -n %ionic_lname +%{_libdir}/libionic*.so.* + %files -n %mana_lname %{_libdir}/libmana*.so.*