diff options
Diffstat (limited to 'drivers/infiniband/hw')
131 files changed, 3823 insertions, 3118 deletions
diff --git a/drivers/infiniband/hw/bnxt_re/Kconfig b/drivers/infiniband/hw/bnxt_re/Kconfig index 0feac5132ce1..6a17f5cdb020 100644 --- a/drivers/infiniband/hw/bnxt_re/Kconfig +++ b/drivers/infiniband/hw/bnxt_re/Kconfig @@ -2,9 +2,7 @@ config INFINIBAND_BNXT_RE tristate "Broadcom Netxtreme HCA support" depends on 64BIT - depends on ETHERNET && NETDEVICES && PCI && INET && DCB - select NET_VENDOR_BROADCOM - select BNXT + depends on INET && DCB && BNXT help This driver supports Broadcom NetXtreme-E 10/25/40/50 gigabit RoCE HCAs. To compile this driver as a module, choose M here: diff --git a/drivers/infiniband/hw/bnxt_re/bnxt_re.h b/drivers/infiniband/hw/bnxt_re/bnxt_re.h index b930ea3dab7a..ba26d8e6a9c2 100644 --- a/drivers/infiniband/hw/bnxt_re/bnxt_re.h +++ b/drivers/infiniband/hw/bnxt_re/bnxt_re.h @@ -138,6 +138,7 @@ struct bnxt_re_dev { #define BNXT_RE_FLAG_QOS_WORK_REG 5 #define BNXT_RE_FLAG_RESOURCES_ALLOCATED 7 #define BNXT_RE_FLAG_RESOURCES_INITIALIZED 8 +#define BNXT_RE_FLAG_ERR_DEVICE_DETACHED 17 #define BNXT_RE_FLAG_ISSUE_ROCE_STATS 29 struct net_device *netdev; unsigned int version, major, minor; diff --git a/drivers/infiniband/hw/bnxt_re/hw_counters.c b/drivers/infiniband/hw/bnxt_re/hw_counters.c index 5f5408cdf008..3e54e1ae75b4 100644 --- a/drivers/infiniband/hw/bnxt_re/hw_counters.c +++ b/drivers/infiniband/hw/bnxt_re/hw_counters.c @@ -114,7 +114,7 @@ static const char * const bnxt_re_stat_name[] = { int bnxt_re_ib_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, - u8 port, int index) + u32 port, int index) { struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); struct ctx_hw_stats *bnxt_re_stats = rdev->qplib_ctx.stats.dma; @@ -235,7 +235,7 @@ int bnxt_re_ib_get_hw_stats(struct ib_device *ibdev, } struct rdma_hw_stats *bnxt_re_ib_alloc_hw_stats(struct ib_device *ibdev, - u8 port_num) + u32 port_num) { BUILD_BUG_ON(ARRAY_SIZE(bnxt_re_stat_name) != BNXT_RE_NUM_COUNTERS); /* We support only per port stats */ diff --git a/drivers/infiniband/hw/bnxt_re/hw_counters.h b/drivers/infiniband/hw/bnxt_re/hw_counters.h index 76399f477e5c..ede048607d6c 100644 --- a/drivers/infiniband/hw/bnxt_re/hw_counters.h +++ b/drivers/infiniband/hw/bnxt_re/hw_counters.h @@ -97,8 +97,8 @@ enum bnxt_re_hw_stats { }; struct rdma_hw_stats *bnxt_re_ib_alloc_hw_stats(struct ib_device *ibdev, - u8 port_num); + u32 port_num); int bnxt_re_ib_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, - u8 port, int index); + u32 port, int index); #endif /* __BNXT_RE_HW_STATS_H__ */ diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index ba515efd4fdc..2efaa80bfbd2 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -189,7 +189,7 @@ int bnxt_re_query_device(struct ib_device *ibdev, } /* Port */ -int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num, +int bnxt_re_query_port(struct ib_device *ibdev, u32 port_num, struct ib_port_attr *port_attr) { struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); @@ -229,7 +229,7 @@ int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num, return 0; } -int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num, +int bnxt_re_get_port_immutable(struct ib_device *ibdev, u32 port_num, struct ib_port_immutable *immutable) { struct ib_port_attr port_attr; @@ -254,7 +254,7 @@ void bnxt_re_query_fw_str(struct ib_device *ibdev, char *str) rdev->dev_attr.fw_ver[2], rdev->dev_attr.fw_ver[3]); } -int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num, +int bnxt_re_query_pkey(struct ib_device *ibdev, u32 port_num, u16 index, u16 *pkey) { struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); @@ -266,7 +266,7 @@ int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num, &rdev->qplib_res.pkey_tbl, index, pkey); } -int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num, +int bnxt_re_query_gid(struct ib_device *ibdev, u32 port_num, int index, union ib_gid *gid) { struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); @@ -374,7 +374,7 @@ int bnxt_re_add_gid(const struct ib_gid_attr *attr, void **context) } enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev, - u8 port_num) + u32 port_num) { return IB_LINK_LAYER_ETHERNET; } diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h index 9a8130b79256..d68671cc6173 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h @@ -149,19 +149,19 @@ static inline u16 bnxt_re_get_rwqe_size(int nsge) int bnxt_re_query_device(struct ib_device *ibdev, struct ib_device_attr *ib_attr, struct ib_udata *udata); -int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num, +int bnxt_re_query_port(struct ib_device *ibdev, u32 port_num, struct ib_port_attr *port_attr); -int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num, +int bnxt_re_get_port_immutable(struct ib_device *ibdev, u32 port_num, struct ib_port_immutable *immutable); void bnxt_re_query_fw_str(struct ib_device *ibdev, char *str); -int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num, +int bnxt_re_query_pkey(struct ib_device *ibdev, u32 port_num, u16 index, u16 *pkey); int bnxt_re_del_gid(const struct ib_gid_attr *attr, void **context); int bnxt_re_add_gid(const struct ib_gid_attr *attr, void **context); -int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num, +int bnxt_re_query_gid(struct ib_device *ibdev, u32 port_num, int index, union ib_gid *gid); enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev, - u8 port_num); + u32 port_num); int bnxt_re_alloc_pd(struct ib_pd *pd, struct ib_udata *udata); int bnxt_re_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata); int bnxt_re_create_ah(struct ib_ah *ah, struct rdma_ah_init_attr *init_attr, diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c index fdb8c2478258..8bfbf0231a9e 100644 --- a/drivers/infiniband/hw/bnxt_re/main.c +++ b/drivers/infiniband/hw/bnxt_re/main.c @@ -81,6 +81,7 @@ static struct workqueue_struct *bnxt_re_wq; static void bnxt_re_remove_device(struct bnxt_re_dev *rdev); static void bnxt_re_dealloc_driver(struct ib_device *ib_dev); static void bnxt_re_stop_irq(void *handle); +static void bnxt_re_dev_stop(struct bnxt_re_dev *rdev); static void bnxt_re_set_drv_mode(struct bnxt_re_dev *rdev, u8 mode) { @@ -221,6 +222,37 @@ static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev) /* for handling bnxt_en callbacks later */ static void bnxt_re_stop(void *p) { + struct bnxt_re_dev *rdev = p; + struct bnxt *bp; + + if (!rdev) + return; + ASSERT_RTNL(); + + /* L2 driver invokes this callback during device error/crash or device + * reset. Current RoCE driver doesn't recover the device in case of + * error. Handle the error by dispatching fatal events to all qps + * ie. by calling bnxt_re_dev_stop and release the MSIx vectors as + * L2 driver want to modify the MSIx table. + */ + bp = netdev_priv(rdev->netdev); + + ibdev_info(&rdev->ibdev, "Handle device stop call from L2 driver"); + /* Check the current device state from L2 structure and move the + * device to detached state if FW_FATAL_COND is set. + * This prevents more commands to HW during clean-up, + * in case the device is already in error. + */ + if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) + set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags); + + bnxt_re_dev_stop(rdev); + bnxt_re_stop_irq(rdev); + /* Move the device states to detached and avoid sending any more + * commands to HW + */ + set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); + set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags); } static void bnxt_re_start(void *p) @@ -234,6 +266,8 @@ static void bnxt_re_sriov_config(void *p, int num_vfs) if (!rdev) return; + if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) + return; rdev->num_vfs = num_vfs; if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx)) { bnxt_re_set_resource_limits(rdev); @@ -427,6 +461,9 @@ static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev, if (!en_dev) return rc; + if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) + return 0; + memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_FREE, -1, -1); @@ -489,6 +526,9 @@ static int bnxt_re_net_stats_ctx_free(struct bnxt_re_dev *rdev, if (!en_dev) return rc; + if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) + return 0; + memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_FREE, -1, -1); @@ -561,24 +601,12 @@ static struct bnxt_re_dev *bnxt_re_from_netdev(struct net_device *netdev) return container_of(ibdev, struct bnxt_re_dev, ibdev); } -static void bnxt_re_dev_unprobe(struct net_device *netdev, - struct bnxt_en_dev *en_dev) -{ - dev_put(netdev); - module_put(en_dev->pdev->driver->driver.owner); -} - static struct bnxt_en_dev *bnxt_re_dev_probe(struct net_device *netdev) { - struct bnxt *bp = netdev_priv(netdev); struct bnxt_en_dev *en_dev; struct pci_dev *pdev; - /* Call bnxt_en's RoCE probe via indirect API */ - if (!bp->ulp_probe) - return ERR_PTR(-EINVAL); - - en_dev = bp->ulp_probe(netdev); + en_dev = bnxt_ulp_probe(netdev); if (IS_ERR(en_dev)) return en_dev; @@ -593,10 +621,6 @@ static struct bnxt_en_dev *bnxt_re_dev_probe(struct net_device *netdev) return ERR_PTR(-ENODEV); } - /* Bump net device reference count */ - if (!try_module_get(pdev->driver->driver.owner)) - return ERR_PTR(-ENODEV); - dev_hold(netdev); return en_dev; @@ -1523,13 +1547,12 @@ fail: static void bnxt_re_dev_unreg(struct bnxt_re_dev *rdev) { - struct bnxt_en_dev *en_dev = rdev->en_dev; struct net_device *netdev = rdev->netdev; bnxt_re_dev_remove(rdev); if (netdev) - bnxt_re_dev_unprobe(netdev, en_dev); + dev_put(netdev); } static int bnxt_re_dev_reg(struct bnxt_re_dev **rdev, struct net_device *netdev) @@ -1551,7 +1574,7 @@ static int bnxt_re_dev_reg(struct bnxt_re_dev **rdev, struct net_device *netdev) *rdev = bnxt_re_dev_add(netdev, en_dev); if (!*rdev) { rc = -ENOMEM; - bnxt_re_dev_unprobe(netdev, en_dev); + dev_put(netdev); goto exit; } exit: diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c index 995d4633b0a1..d4d4959c2434 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c @@ -2784,6 +2784,7 @@ do_rq: dev_err(&cq->hwq.pdev->dev, "FP: CQ Processed terminal reported rq_cons_idx 0x%x exceeds max 0x%x\n", cqe_cons, rq->max_wqe); + rc = -EINVAL; goto done; } diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c index 441eb421e5e5..5d384def5e5f 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c @@ -212,6 +212,10 @@ int bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw, u8 opcode, retry_cnt = 0xFF; int rc = 0; + /* Prevent posting if f/w is not in a state to process */ + if (test_bit(ERR_DEVICE_DETACHED, &rcfw->cmdq.flags)) + return 0; + do { opcode = req->opcode; rc = __send_message(rcfw, req, resp, sb, is_block); diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h index 5f2f0a5a3560..9474c0046582 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h +++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h @@ -138,6 +138,8 @@ struct bnxt_qplib_qp_node { #define FIRMWARE_INITIALIZED_FLAG (0) #define FIRMWARE_FIRST_FLAG (31) #define FIRMWARE_TIMED_OUT (3) +#define ERR_DEVICE_DETACHED (4) + struct bnxt_qplib_cmdq_mbox { struct bnxt_qplib_reg_desc reg; void __iomem *prod; diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c index fa7878336100..3ca47004b752 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_res.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c @@ -854,6 +854,7 @@ static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res *res, unmap_io: pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem); + dpit->dbr_bar_reg_iomem = NULL; return -ENOMEM; } diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c index e42c812e74c3..291471d12197 100644 --- a/drivers/infiniband/hw/cxgb4/cm.c +++ b/drivers/infiniband/hw/cxgb4/cm.c @@ -145,7 +145,7 @@ static void connect_reply_upcall(struct c4iw_ep *ep, int status); static int sched(struct c4iw_dev *dev, struct sk_buff *skb); static LIST_HEAD(timeout_list); -static spinlock_t timeout_lock; +static DEFINE_SPINLOCK(timeout_lock); static void deref_cm_id(struct c4iw_ep_common *epc) { @@ -4452,7 +4452,6 @@ c4iw_handler_func c4iw_handlers[NUM_CPL_CMDS] = { int __init c4iw_cm_init(void) { - spin_lock_init(&timeout_lock); skb_queue_head_init(&rxq); workq = alloc_ordered_workqueue("iw_cxgb4", WQ_MEM_RECLAIM); diff --git a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h index f85477f3b037..cdec5deb37a1 100644 --- a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h +++ b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h @@ -341,11 +341,6 @@ static inline struct c4iw_dev *to_c4iw_dev(struct ib_device *ibdev) return container_of(ibdev, struct c4iw_dev, ibdev); } -static inline struct c4iw_dev *rdev_to_c4iw_dev(struct c4iw_rdev *rdev) -{ - return container_of(rdev, struct c4iw_dev, rdev); -} - static inline struct c4iw_cq *get_chp(struct c4iw_dev *rhp, u32 cqid) { return xa_load(&rhp->cqs, cqid); @@ -659,12 +654,6 @@ static inline u32 c4iw_ib_to_tpt_access(int a) FW_RI_MEM_ACCESS_LOCAL_READ; } -static inline u32 c4iw_ib_to_tpt_bind_access(int acc) -{ - return (acc & IB_ACCESS_REMOTE_WRITE ? FW_RI_MEM_ACCESS_REM_WRITE : 0) | - (acc & IB_ACCESS_REMOTE_READ ? FW_RI_MEM_ACCESS_REM_READ : 0); -} - enum c4iw_mmid_state { C4IW_STAG_STATE_VALID, C4IW_STAG_STATE_INVALID diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c index 1f1f856f8715..3f1893e180dd 100644 --- a/drivers/infiniband/hw/cxgb4/provider.c +++ b/drivers/infiniband/hw/cxgb4/provider.c @@ -237,12 +237,12 @@ static int c4iw_allocate_pd(struct ib_pd *pd, struct ib_udata *udata) return 0; } -static int c4iw_query_gid(struct ib_device *ibdev, u8 port, int index, +static int c4iw_query_gid(struct ib_device *ibdev, u32 port, int index, union ib_gid *gid) { struct c4iw_dev *dev; - pr_debug("ibdev %p, port %d, index %d, gid %p\n", + pr_debug("ibdev %p, port %u, index %d, gid %p\n", ibdev, port, index, gid); if (!port) return -EINVAL; @@ -295,7 +295,7 @@ static int c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *pro return 0; } -static int c4iw_query_port(struct ib_device *ibdev, u8 port, +static int c4iw_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props) { int ret = 0; @@ -378,7 +378,7 @@ static const char * const names[] = { }; static struct rdma_hw_stats *c4iw_alloc_stats(struct ib_device *ibdev, - u8 port_num) + u32 port_num) { BUILD_BUG_ON(ARRAY_SIZE(names) != NR_COUNTERS); @@ -391,7 +391,7 @@ static struct rdma_hw_stats *c4iw_alloc_stats(struct ib_device *ibdev, static int c4iw_get_mib(struct ib_device *ibdev, struct rdma_hw_stats *stats, - u8 port, int index) + u32 port, int index) { struct tp_tcp_stats v4, v6; struct c4iw_dev *c4iw_dev = to_c4iw_dev(ibdev); @@ -420,7 +420,7 @@ static const struct attribute_group c4iw_attr_group = { .attrs = c4iw_class_attributes, }; -static int c4iw_port_immutable(struct ib_device *ibdev, u8 port_num, +static int c4iw_port_immutable(struct ib_device *ibdev, u32 port_num, struct ib_port_immutable *immutable) { struct ib_port_attr attr; diff --git a/drivers/infiniband/hw/cxgb4/resource.c b/drivers/infiniband/hw/cxgb4/resource.c index 5c95c789f302..e800e8e8bed5 100644 --- a/drivers/infiniband/hw/cxgb4/resource.c +++ b/drivers/infiniband/hw/cxgb4/resource.c @@ -216,7 +216,7 @@ u32 c4iw_get_qpid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx) goto out; entry->qid = qid; list_add_tail(&entry->entry, &uctx->cqids); - for (i = qid; i & rdev->qpmask; i++) { + for (i = qid + 1; i & rdev->qpmask; i++) { entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) goto out; diff --git a/drivers/infiniband/hw/cxgb4/t4.h b/drivers/infiniband/hw/cxgb4/t4.h index b170817b2741..c3b0e2896475 100644 --- a/drivers/infiniband/hw/cxgb4/t4.h +++ b/drivers/infiniband/hw/cxgb4/t4.h @@ -487,11 +487,6 @@ static inline int t4_rq_empty(struct t4_wq *wq) return wq->rq.in_use == 0; } -static inline int t4_rq_full(struct t4_wq *wq) -{ - return wq->rq.in_use == (wq->rq.size - 1); -} - static inline u32 t4_rq_avail(struct t4_wq *wq) { return wq->rq.size - 1 - wq->rq.in_use; @@ -534,11 +529,6 @@ static inline int t4_sq_empty(struct t4_wq *wq) return wq->sq.in_use == 0; } -static inline int t4_sq_full(struct t4_wq *wq) -{ - return wq->sq.in_use == (wq->sq.size - 1); -} - static inline u32 t4_sq_avail(struct t4_wq *wq) { return wq->sq.size - 1 - wq->sq.in_use; @@ -679,11 +669,6 @@ static inline void t4_enable_wq_db(struct t4_wq *wq) wq->rq.queue[wq->rq.size].status.db_off = 0; } -static inline int t4_wq_db_enabled(struct t4_wq *wq) -{ - return !wq->rq.queue[wq->rq.size].status.db_off; -} - enum t4_cq_flags { CQ_ARMED = 1, }; @@ -817,19 +802,6 @@ static inline int t4_next_hw_cqe(struct t4_cq *cq, struct t4_cqe **cqe) return ret; } -static inline struct t4_cqe *t4_next_sw_cqe(struct t4_cq *cq) -{ - if (cq->sw_in_use == cq->size) { - pr_warn("%s cxgb4 sw cq overflow cqid %u\n", - __func__, cq->cqid); - cq->error = 1; - return NULL; - } - if (cq->sw_in_use) - return &cq->sw_queue[cq->sw_cidx]; - return NULL; -} - static inline int t4_next_cqe(struct t4_cq *cq, struct t4_cqe **cqe) { int ret = 0; @@ -843,11 +815,6 @@ static inline int t4_next_cqe(struct t4_cq *cq, struct t4_cqe **cqe) return ret; } -static inline int t4_cq_in_error(struct t4_cq *cq) -{ - return *cq->qp_errp; -} - static inline void t4_set_cq_in_error(struct t4_cq *cq) { *cq->qp_errp = 1; diff --git a/drivers/infiniband/hw/efa/efa.h b/drivers/infiniband/hw/efa/efa.h index e5d9712e98c4..ea322cec27d2 100644 --- a/drivers/infiniband/hw/efa/efa.h +++ b/drivers/infiniband/hw/efa/efa.h @@ -120,14 +120,14 @@ struct efa_ah { int efa_query_device(struct ib_device *ibdev, struct ib_device_attr *props, struct ib_udata *udata); -int efa_query_port(struct ib_device *ibdev, u8 port, +int efa_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props); int efa_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr); -int efa_query_gid(struct ib_device *ibdev, u8 port, int index, +int efa_query_gid(struct ib_device *ibdev, u32 port, int index, union ib_gid *gid); -int efa_query_pkey(struct ib_device *ibdev, u8 port, u16 index, +int efa_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey); int efa_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); int efa_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); @@ -142,7 +142,7 @@ struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length, u64 virt_addr, int access_flags, struct ib_udata *udata); int efa_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata); -int efa_get_port_immutable(struct ib_device *ibdev, u8 port_num, +int efa_get_port_immutable(struct ib_device *ibdev, u32 port_num, struct ib_port_immutable *immutable); int efa_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata); void efa_dealloc_ucontext(struct ib_ucontext *ibucontext); @@ -156,9 +156,9 @@ int efa_destroy_ah(struct ib_ah *ibah, u32 flags); int efa_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_udata *udata); enum rdma_link_layer efa_port_link_layer(struct ib_device *ibdev, - u8 port_num); -struct rdma_hw_stats *efa_alloc_hw_stats(struct ib_device *ibdev, u8 port_num); + u32 port_num); +struct rdma_hw_stats *efa_alloc_hw_stats(struct ib_device *ibdev, u32 port_num); int efa_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, - u8 port_num, int index); + u32 port_num, int index); #endif /* _EFA_H_ */ diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c index 0f578734bddb..816cfd65b7ac 100644 --- a/drivers/infiniband/hw/efa/efa_main.c +++ b/drivers/infiniband/hw/efa/efa_main.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause /* - * Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved. + * Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All rights reserved. */ #include <linux/module.h> @@ -209,11 +209,11 @@ static void efa_set_host_info(struct efa_dev *dev) if (!hinf) return; - strlcpy(hinf->os_dist_str, utsname()->release, - min(sizeof(hinf->os_dist_str), sizeof(utsname()->release))); + strscpy(hinf->os_dist_str, utsname()->release, + sizeof(hinf->os_dist_str)); hinf->os_type = EFA_ADMIN_OS_LINUX; - strlcpy(hinf->kernel_ver_str, utsname()->version, - min(sizeof(hinf->kernel_ver_str), sizeof(utsname()->version))); + strscpy(hinf->kernel_ver_str, utsname()->version, + sizeof(hinf->kernel_ver_str)); hinf->kernel_ver = LINUX_VERSION_CODE; EFA_SET(&hinf->driver_ver, EFA_ADMIN_HOST_INFO_DRIVER_MAJOR, 0); EFA_SET(&hinf->driver_ver, EFA_ADMIN_HOST_INFO_DRIVER_MINOR, 0); diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 479b604e533a..51572f1dc611 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -247,7 +247,7 @@ int efa_query_device(struct ib_device *ibdev, return 0; } -int efa_query_port(struct ib_device *ibdev, u8 port, +int efa_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props) { struct efa_dev *dev = to_edev(ibdev); @@ -319,7 +319,7 @@ int efa_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, return 0; } -int efa_query_gid(struct ib_device *ibdev, u8 port, int index, +int efa_query_gid(struct ib_device *ibdev, u32 port, int index, union ib_gid *gid) { struct efa_dev *dev = to_edev(ibdev); @@ -329,7 +329,7 @@ int efa_query_gid(struct ib_device *ibdev, u8 port, int index, return 0; } -int efa_query_pkey(struct ib_device *ibdev, u8 port, u16 index, +int efa_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) { if (index > 0) @@ -1619,7 +1619,7 @@ int efa_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) return 0; } -int efa_get_port_immutable(struct ib_device *ibdev, u8 port_num, +int efa_get_port_immutable(struct ib_device *ibdev, u32 port_num, struct ib_port_immutable *immutable) { struct ib_port_attr attr; @@ -1904,7 +1904,7 @@ int efa_destroy_ah(struct ib_ah *ibah, u32 flags) return 0; } -struct rdma_hw_stats *efa_alloc_hw_stats(struct ib_device *ibdev, u8 port_num) +struct rdma_hw_stats *efa_alloc_hw_stats(struct ib_device *ibdev, u32 port_num) { return rdma_alloc_hw_stats_struct(efa_stats_names, ARRAY_SIZE(efa_stats_names), @@ -1912,7 +1912,7 @@ struct rdma_hw_stats *efa_alloc_hw_stats(struct ib_device *ibdev, u8 port_num) } int efa_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, - u8 port_num, int index) + u32 port_num, int index) { struct efa_com_get_stats_params params = {}; union efa_com_get_stats_result result; @@ -1981,7 +1981,7 @@ int efa_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, } enum rdma_link_layer efa_port_link_layer(struct ib_device *ibdev, - u8 port_num) + u32 port_num) { return IB_LINK_LAYER_UNSPECIFIED; } diff --git a/drivers/infiniband/hw/hfi1/affinity.c b/drivers/infiniband/hw/hfi1/affinity.c index 04b1e8f021f6..16543f717527 100644 --- a/drivers/infiniband/hw/hfi1/affinity.c +++ b/drivers/infiniband/hw/hfi1/affinity.c @@ -962,7 +962,6 @@ void hfi1_put_irq_affinity(struct hfi1_devdata *dd, struct hfi1_msix_entry *msix) { struct cpu_mask_set *set = NULL; - struct hfi1_ctxtdata *rcd; struct hfi1_affinity_node *entry; mutex_lock(&node_affinity.lock); @@ -976,14 +975,15 @@ void hfi1_put_irq_affinity(struct hfi1_devdata *dd, case IRQ_GENERAL: /* Don't do accounting for general contexts */ break; - case IRQ_RCVCTXT: - rcd = (struct hfi1_ctxtdata *)msix->arg; + case IRQ_RCVCTXT: { + struct hfi1_ctxtdata *rcd = msix->arg; + /* Don't do accounting for control contexts */ if (rcd->ctxt != HFI1_CTRL_CTXT) set = &entry->rcv_intr; break; + } case IRQ_NETDEVCTXT: - rcd = (struct hfi1_ctxtdata *)msix->arg; set = &entry->def_intr; break; default: diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c index 993cbf37e0b9..5eeae8df415b 100644 --- a/drivers/infiniband/hw/hfi1/chip.c +++ b/drivers/infiniband/hw/hfi1/chip.c @@ -1322,7 +1322,7 @@ CNTR_ELEM(#name, \ access_ibp_##cntr) /** - * hfi_addr_from_offset - return addr for readq/writeq + * hfi1_addr_from_offset - return addr for readq/writeq * @dd: the dd device * @offset: the offset of the CSR within bar0 * @@ -8316,7 +8316,7 @@ static void is_interrupt(struct hfi1_devdata *dd, unsigned int source) } /** - * gerneral_interrupt() - General interrupt handler + * general_interrupt - General interrupt handler * @irq: MSIx IRQ vector * @data: hfi1 devdata * @@ -15243,8 +15243,8 @@ int hfi1_init_dd(struct hfi1_devdata *dd) (dd->revision >> CCE_REVISION_SW_SHIFT) & CCE_REVISION_SW_MASK); - /* alloc netdev data */ - ret = hfi1_netdev_alloc(dd); + /* alloc VNIC/AIP rx data */ + ret = hfi1_alloc_rx(dd); if (ret) goto bail_cleanup; @@ -15348,7 +15348,7 @@ bail_clear_intr: hfi1_comp_vectors_clean_up(dd); msix_clean_up_interrupts(dd); bail_cleanup: - hfi1_netdev_free(dd); + hfi1_free_rx(dd); hfi1_pcie_ddcleanup(dd); bail_free: hfi1_free_devdata(dd); diff --git a/drivers/infiniband/hw/hfi1/chip.h b/drivers/infiniband/hw/hfi1/chip.h index 2c6f2de74d4d..ac26649d4463 100644 --- a/drivers/infiniband/hw/hfi1/chip.h +++ b/drivers/infiniband/hw/hfi1/chip.h @@ -822,11 +822,6 @@ int acquire_lcb_access(struct hfi1_devdata *dd, int sleep_ok); int release_lcb_access(struct hfi1_devdata *dd, int sleep_ok); #define LCB_START DC_LCB_CSRS #define LCB_END DC_8051_CSRS /* next block is 8051 */ -static inline int is_lcb_offset(u32 offset) -{ - return (offset >= LCB_START && offset < LCB_END); -} - extern uint num_vls; extern uint disable_integrity; diff --git a/drivers/infiniband/hw/hfi1/driver.c b/drivers/infiniband/hw/hfi1/driver.c index 0b64aa87ab73..f88bb4af245f 100644 --- a/drivers/infiniband/hw/hfi1/driver.c +++ b/drivers/infiniband/hw/hfi1/driver.c @@ -1026,7 +1026,7 @@ static bool __set_armed_to_active(struct hfi1_packet *packet) } /** - * armed to active - the fast path for armed to active + * set_armed_to_active - the fast path for armed to active * @packet: the packet structure * * Return true if packet processing needs to bail. diff --git a/drivers/infiniband/hw/hfi1/exp_rcv.c b/drivers/infiniband/hw/hfi1/exp_rcv.c index 91f13140ddf2..a414214f6035 100644 --- a/drivers/infiniband/hw/hfi1/exp_rcv.c +++ b/drivers/infiniband/hw/hfi1/exp_rcv.c @@ -49,7 +49,7 @@ #include "trace.h" /** - * exp_tid_group_init - initialize exp_tid_set + * hfi1_exp_tid_set_init - initialize exp_tid_set * @set: the set */ static void hfi1_exp_tid_set_init(struct exp_tid_set *set) @@ -70,7 +70,7 @@ void hfi1_exp_tid_group_init(struct hfi1_ctxtdata *rcd) } /** - * alloc_ctxt_rcv_groups - initialize expected receive groups + * hfi1_alloc_ctxt_rcv_groups - initialize expected receive groups * @rcd: the context to add the groupings to */ int hfi1_alloc_ctxt_rcv_groups(struct hfi1_ctxtdata *rcd) @@ -100,7 +100,7 @@ int hfi1_alloc_ctxt_rcv_groups(struct hfi1_ctxtdata *rcd) } /** - * free_ctxt_rcv_groups - free expected receive groups + * hfi1_free_ctxt_rcv_groups - free expected receive groups * @rcd: the context to free * * The routine dismantles the expect receive linked diff --git a/drivers/infiniband/hw/hfi1/firmware.c b/drivers/infiniband/hw/hfi1/firmware.c index 0e83d4b61e46..2cf102b5abd4 100644 --- a/drivers/infiniband/hw/hfi1/firmware.c +++ b/drivers/infiniband/hw/hfi1/firmware.c @@ -1916,6 +1916,7 @@ int parse_platform_config(struct hfi1_devdata *dd) dd_dev_err(dd, "%s: Failed CRC check at offset %ld\n", __func__, (ptr - (u32 *)dd->platform_config.data)); + ret = -EINVAL; goto bail; } /* Jump the CRC DWORD */ diff --git a/drivers/infiniband/hw/hfi1/hfi.h b/drivers/infiniband/hw/hfi1/hfi.h index 2a9a040569eb..867ae0b1aa95 100644 --- a/drivers/infiniband/hw/hfi1/hfi.h +++ b/drivers/infiniband/hw/hfi1/hfi.h @@ -69,7 +69,6 @@ #include <rdma/ib_hdrs.h> #include <rdma/opa_addr.h> #include <linux/rhashtable.h> -#include <linux/netdevice.h> #include <rdma/rdma_vt.h> #include "chip_registers.h" @@ -717,12 +716,6 @@ static inline void incr_cntr64(u64 *cntr) (*cntr)++; } -static inline void incr_cntr32(u32 *cntr) -{ - if (*cntr < (u32)-1LL) - (*cntr)++; -} - #define MAX_NAME_SIZE 64 struct hfi1_msix_entry { enum irq_type type; @@ -864,7 +857,7 @@ struct hfi1_pportdata { u8 rx_pol_inv; u8 hw_pidx; /* physical port index */ - u8 port; /* IB port number and index into dd->pports - 1 */ + u32 port; /* IB port number and index into dd->pports - 1 */ /* type of neighbor node */ u8 neighbor_type; u8 neighbor_normal; @@ -1066,6 +1059,7 @@ struct sdma_vl_map; #define SERIAL_MAX 16 /* length of the serial number */ typedef int (*send_routine)(struct rvt_qp *, struct hfi1_pkt_state *, u64); +struct hfi1_netdev_rx; struct hfi1_devdata { struct hfi1_ibdev verbs_dev; /* must be first */ /* pointers to related structs for this device */ @@ -1408,7 +1402,7 @@ struct hfi1_devdata { /* Lock to protect IRQ SRC register access */ spinlock_t irq_src_lock; int vnic_num_vports; - struct net_device *dummy_netdev; + struct hfi1_netdev_rx *netdev_rx; struct hfi1_affinity_node *affinity_entry; /* Keeps track of IPoIB RSM rule users */ @@ -1480,7 +1474,7 @@ int hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, int numa, struct hfi1_ctxtdata **rcd); void hfi1_free_ctxt(struct hfi1_ctxtdata *rcd); void hfi1_init_pportdata(struct pci_dev *pdev, struct hfi1_pportdata *ppd, - struct hfi1_devdata *dd, u8 hw_pidx, u8 port); + struct hfi1_devdata *dd, u8 hw_pidx, u32 port); void hfi1_free_ctxtdata(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd); int hfi1_rcd_put(struct hfi1_ctxtdata *rcd); int hfi1_rcd_get(struct hfi1_ctxtdata *rcd); @@ -1976,10 +1970,10 @@ static inline struct hfi1_ibdev *dev_from_rdi(struct rvt_dev_info *rdi) return container_of(rdi, struct hfi1_ibdev, rdi); } -static inline struct hfi1_ibport *to_iport(struct ib_device *ibdev, u8 port) +static inline struct hfi1_ibport *to_iport(struct ib_device *ibdev, u32 port) { struct hfi1_devdata *dd = dd_from_ibdev(ibdev); - unsigned pidx = port - 1; /* IB number port from 1, hdw from 0 */ + u32 pidx = port - 1; /* IB number port from 1, hdw from 0 */ WARN_ON(pidx >= dd->num_pports); return &dd->pport[pidx].ibport_data; @@ -2198,7 +2192,7 @@ extern const struct attribute_group ib_hfi1_attr_group; int hfi1_device_create(struct hfi1_devdata *dd); void hfi1_device_remove(struct hfi1_devdata *dd); -int hfi1_create_port_files(struct ib_device *ibdev, u8 port_num, +int hfi1_create_port_files(struct ib_device *ibdev, u32 port_num, struct kobject *kobj); int hfi1_verbs_register_sysfs(struct hfi1_devdata *dd); void hfi1_verbs_unregister_sysfs(struct hfi1_devdata *dd); diff --git a/drivers/infiniband/hw/hfi1/init.c b/drivers/infiniband/hw/hfi1/init.c index 786c6316273f..e3a8a420c045 100644 --- a/drivers/infiniband/hw/hfi1/init.c +++ b/drivers/infiniband/hw/hfi1/init.c @@ -627,7 +627,7 @@ static enum hrtimer_restart cca_timer_fn(struct hrtimer *t) * Common code for initializing the physical port structure. */ void hfi1_init_pportdata(struct pci_dev *pdev, struct hfi1_pportdata *ppd, - struct hfi1_devdata *dd, u8 hw_pidx, u8 port) + struct hfi1_devdata *dd, u8 hw_pidx, u32 port) { int i; uint default_pkey_idx; @@ -1775,7 +1775,7 @@ static void remove_one(struct pci_dev *pdev) hfi1_unregister_ib_device(dd); /* free netdev data */ - hfi1_netdev_free(dd); + hfi1_free_rx(dd); /* * Disable the IB link, disable interrupts on the device, @@ -1860,7 +1860,8 @@ bail: } /** - * allocate eager buffers, both kernel and user contexts. + * hfi1_setup_eagerbufs - llocate eager buffers, both kernel and user + * contexts. * @rcd: the context we are setting up. * * Allocate the eager TID buffers and program them into hip. diff --git a/drivers/infiniband/hw/hfi1/iowait.h b/drivers/infiniband/hw/hfi1/iowait.h index d580aa17ae37..cda81a7843c2 100644 --- a/drivers/infiniband/hw/hfi1/iowait.h +++ b/drivers/infiniband/hw/hfi1/iowait.h @@ -321,7 +321,7 @@ static inline void iowait_drain_wakeup(struct iowait *wait) /** * iowait_get_txhead() - get packet off of iowait list * - * @wait iowait_work struture + * @wait: iowait_work structure */ static inline struct sdma_txreq *iowait_get_txhead(struct iowait_work *wait) { diff --git a/drivers/infiniband/hw/hfi1/ipoib.h b/drivers/infiniband/hw/hfi1/ipoib.h index f650cac9d424..2cff38b105ac 100644 --- a/drivers/infiniband/hw/hfi1/ipoib.h +++ b/drivers/infiniband/hw/hfi1/ipoib.h @@ -52,8 +52,9 @@ union hfi1_ipoib_flow { * @producer_lock: producer sync lock * @consumer_lock: consumer sync lock */ +struct ipoib_txreq; struct hfi1_ipoib_circ_buf { - void **items; + struct ipoib_txreq **items; unsigned long head; unsigned long tail; unsigned long max_items; @@ -125,10 +126,10 @@ hfi1_ipoib_priv(const struct net_device *dev) return &((struct hfi1_ipoib_rdma_netdev *)netdev_priv(dev))->dev_priv; } -int hfi1_ipoib_send_dma(struct net_device *dev, - struct sk_buff *skb, - struct ib_ah *address, - u32 dqpn); +int hfi1_ipoib_send(struct net_device *dev, + struct sk_buff *skb, + struct ib_ah *address, + u32 dqpn); int hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv *priv); void hfi1_ipoib_txreq_deinit(struct hfi1_ipoib_dev_priv *priv); @@ -143,8 +144,10 @@ struct sk_buff *hfi1_ipoib_prepare_skb(struct hfi1_netdev_rxq *rxq, int size, void *data); int hfi1_ipoib_rn_get_params(struct ib_device *device, - u8 port_num, + u32 port_num, enum rdma_netdev_t type, struct rdma_netdev_alloc_params *params); +void hfi1_ipoib_tx_timeout(struct net_device *dev, unsigned int q); + #endif /* _IPOIB_H */ diff --git a/drivers/infiniband/hw/hfi1/ipoib_main.c b/drivers/infiniband/hw/hfi1/ipoib_main.c index 3242290eb6a7..e594a961f513 100644 --- a/drivers/infiniband/hw/hfi1/ipoib_main.c +++ b/drivers/infiniband/hw/hfi1/ipoib_main.c @@ -101,14 +101,6 @@ static const struct net_device_ops hfi1_ipoib_netdev_ops = { .ndo_get_stats64 = dev_get_tstats64, }; -static int hfi1_ipoib_send(struct net_device *dev, - struct sk_buff *skb, - struct ib_ah *address, - u32 dqpn) -{ - return hfi1_ipoib_send_dma(dev, skb, address, dqpn); -} - static int hfi1_ipoib_mcast_attach(struct net_device *dev, struct ib_device *device, union ib_gid *mgid, @@ -194,7 +186,7 @@ static void hfi1_ipoib_set_id(struct net_device *dev, int id) } static int hfi1_ipoib_setup_rn(struct ib_device *device, - u8 port_num, + u32 port_num, struct net_device *netdev, void *param) { @@ -204,6 +196,7 @@ static int hfi1_ipoib_setup_rn(struct ib_device *device, int rc; rn->send = hfi1_ipoib_send; + rn->tx_timeout = hfi1_ipoib_tx_timeout; rn->attach_mcast = hfi1_ipoib_mcast_attach; rn->detach_mcast = hfi1_ipoib_mcast_detach; rn->set_id = hfi1_ipoib_set_id; @@ -243,7 +236,7 @@ static int hfi1_ipoib_setup_rn(struct ib_device *device, } int hfi1_ipoib_rn_get_params(struct ib_device *device, - u8 port_num, + u32 port_num, enum rdma_netdev_t type, struct rdma_netdev_alloc_params *params) { diff --git a/drivers/infiniband/hw/hfi1/ipoib_tx.c b/drivers/infiniband/hw/hfi1/ipoib_tx.c index edd4eeac8dd1..993f9838b6c8 100644 --- a/drivers/infiniband/hw/hfi1/ipoib_tx.c +++ b/drivers/infiniband/hw/hfi1/ipoib_tx.c @@ -15,6 +15,7 @@ #include "verbs.h" #include "trace_ibhdrs.h" #include "ipoib.h" +#include "trace_tx.h" /* Add a convenience helper */ #define CIRC_ADD(val, add, size) (((val) + (add)) & ((size) - 1)) @@ -63,12 +64,14 @@ static u64 hfi1_ipoib_used(struct hfi1_ipoib_txq *txq) static void hfi1_ipoib_stop_txq(struct hfi1_ipoib_txq *txq) { + trace_hfi1_txq_stop(txq); if (atomic_inc_return(&txq->stops) == 1) netif_stop_subqueue(txq->priv->netdev, txq->q_idx); } static void hfi1_ipoib_wake_txq(struct hfi1_ipoib_txq *txq) { + trace_hfi1_txq_wake(txq); if (atomic_dec_and_test(&txq->stops)) netif_wake_subqueue(txq->priv->netdev, txq->q_idx); } @@ -89,8 +92,10 @@ static void hfi1_ipoib_check_queue_depth(struct hfi1_ipoib_txq *txq) { ++txq->sent_txreqs; if (hfi1_ipoib_used(txq) >= hfi1_ipoib_ring_hwat(txq) && - !atomic_xchg(&txq->ring_full, 1)) + !atomic_xchg(&txq->ring_full, 1)) { + trace_hfi1_txq_full(txq); hfi1_ipoib_stop_txq(txq); + } } static void hfi1_ipoib_check_queue_stopped(struct hfi1_ipoib_txq *txq) @@ -112,8 +117,10 @@ static void hfi1_ipoib_check_queue_stopped(struct hfi1_ipoib_txq *txq) * to protect against ring overflow. */ if (hfi1_ipoib_used(txq) < hfi1_ipoib_ring_lwat(txq) && - atomic_xchg(&txq->ring_full, 0)) + atomic_xchg(&txq->ring_full, 0)) { + trace_hfi1_txq_xmit_unstopped(txq); hfi1_ipoib_wake_txq(txq); + } } static void hfi1_ipoib_free_tx(struct ipoib_txreq *tx, int budget) @@ -202,7 +209,7 @@ static void hfi1_ipoib_add_tx(struct ipoib_txreq *tx) /* Finish storing txreq before incrementing head. */ smp_store_release(&tx_ring->head, CIRC_ADD(head, 1, max_tx)); - napi_schedule(tx->txq->napi); + napi_schedule_irqoff(tx->txq->napi); } else { struct hfi1_ipoib_txq *txq = tx->txq; struct hfi1_ipoib_dev_priv *priv = tx->priv; @@ -405,6 +412,7 @@ static struct ipoib_txreq *hfi1_ipoib_send_dma_common(struct net_device *dev, sdma_select_engine_sc(priv->dd, txp->flow.tx_queue, txp->flow.sc5); + trace_hfi1_flow_switch(txp->txq); } return tx; @@ -525,6 +533,7 @@ static int hfi1_ipoib_send_dma_list(struct net_device *dev, if (txq->flow.as_int != txp->flow.as_int) { int ret; + trace_hfi1_flow_flush(txq); ret = hfi1_ipoib_flush_tx_list(dev, txq); if (unlikely(ret)) { if (ret == -EBUSY) @@ -572,10 +581,10 @@ static u8 hfi1_ipoib_calc_entropy(struct sk_buff *skb) return (u8)skb_get_queue_mapping(skb); } -int hfi1_ipoib_send_dma(struct net_device *dev, - struct sk_buff *skb, - struct ib_ah *address, - u32 dqpn) +int hfi1_ipoib_send(struct net_device *dev, + struct sk_buff *skb, + struct ib_ah *address, + u32 dqpn) { struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev); struct ipoib_txparms txp; @@ -635,8 +644,10 @@ static int hfi1_ipoib_sdma_sleep(struct sdma_engine *sde, /* came from non-list submit */ list_add_tail(&txreq->list, &txq->tx_list); if (list_empty(&txq->wait.list)) { - if (!atomic_xchg(&txq->no_desc, 1)) + if (!atomic_xchg(&txq->no_desc, 1)) { + trace_hfi1_txq_queued(txq); hfi1_ipoib_stop_txq(txq); + } iowait_queue(pkts_sent, wait->iow, &sde->dmawait); } @@ -659,6 +670,7 @@ static void hfi1_ipoib_sdma_wakeup(struct iowait *wait, int reason) struct hfi1_ipoib_txq *txq = container_of(wait, struct hfi1_ipoib_txq, wait); + trace_hfi1_txq_wakeup(txq); if (likely(txq->priv->netdev->reg_state == NETREG_REGISTERED)) iowait_schedule(wait, system_highpri_wq, WORK_CPU_UNBOUND); } @@ -702,14 +714,14 @@ int hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv *priv) priv->tx_napis = kcalloc_node(dev->num_tx_queues, sizeof(struct napi_struct), - GFP_ATOMIC, + GFP_KERNEL, priv->dd->node); if (!priv->tx_napis) goto free_txreq_cache; priv->txqs = kcalloc_node(dev->num_tx_queues, sizeof(struct hfi1_ipoib_txq), - GFP_ATOMIC, + GFP_KERNEL, priv->dd->node); if (!priv->txqs) goto free_tx_napis; @@ -741,9 +753,9 @@ int hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv *priv) priv->dd->node); txq->tx_ring.items = - vzalloc_node(array_size(tx_ring_size, - sizeof(struct ipoib_txreq)), - priv->dd->node); + kcalloc_node(tx_ring_size, + sizeof(struct ipoib_txreq *), + GFP_KERNEL, priv->dd->node); if (!txq->tx_ring.items) goto free_txqs; @@ -764,7 +776,7 @@ free_txqs: struct hfi1_ipoib_txq *txq = &priv->txqs[i]; netif_napi_del(txq->napi); - vfree(txq->tx_ring.items); + kfree(txq->tx_ring.items); } kfree(priv->txqs); @@ -817,7 +829,7 @@ void hfi1_ipoib_txreq_deinit(struct hfi1_ipoib_dev_priv *priv) hfi1_ipoib_drain_tx_list(txq); netif_napi_del(txq->napi); (void)hfi1_ipoib_drain_tx_ring(txq, txq->tx_ring.max_items); - vfree(txq->tx_ring.items); + kfree(txq->tx_ring.items); } kfree(priv->txqs); @@ -854,3 +866,32 @@ void hfi1_ipoib_napi_tx_disable(struct net_device *dev) (void)hfi1_ipoib_drain_tx_ring(txq, txq->tx_ring.max_items); } } + +void hfi1_ipoib_tx_timeout(struct net_device *dev, unsigned int q) +{ + struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev); + struct hfi1_ipoib_txq *txq = &priv->txqs[q]; + u64 completed = atomic64_read(&txq->complete_txreqs); + + dd_dev_info(priv->dd, "timeout txq %llx q %u stopped %u stops %d no_desc %d ring_full %d\n", + (unsigned long long)txq, q, + __netif_subqueue_stopped(dev, txq->q_idx), + atomic_read(&txq->stops), + atomic_read(&txq->no_desc), + atomic_read(&txq->ring_full)); + dd_dev_info(priv->dd, "sde %llx engine %u\n", + (unsigned long long)txq->sde, + txq->sde ? txq->sde->this_idx : 0); + dd_dev_info(priv->dd, "flow %x\n", txq->flow.as_int); + dd_dev_info(priv->dd, "sent %llu completed %llu used %llu\n", + txq->sent_txreqs, completed, hfi1_ipoib_used(txq)); + dd_dev_info(priv->dd, "tx_queue_len %u max_items %lu\n", + dev->tx_queue_len, txq->tx_ring.max_items); + dd_dev_info(priv->dd, "head %lu tail %lu\n", + txq->tx_ring.head, txq->tx_ring.tail); + dd_dev_info(priv->dd, "wait queued %u\n", + !list_empty(&txq->wait.list)); + dd_dev_info(priv->dd, "tx_list empty %u\n", + list_empty(&txq->tx_list)); +} + diff --git a/drivers/infiniband/hw/hfi1/mad.c b/drivers/infiniband/hw/hfi1/mad.c index e2f2f7847aed..1fe5e702f31d 100644 --- a/drivers/infiniband/hw/hfi1/mad.c +++ b/drivers/infiniband/hw/hfi1/mad.c @@ -108,7 +108,7 @@ static u16 hfi1_lookup_pkey_value(struct hfi1_ibport *ibp, int pkey_idx) return 0; } -void hfi1_event_pkey_change(struct hfi1_devdata *dd, u8 port) +void hfi1_event_pkey_change(struct hfi1_devdata *dd, u32 port) { struct ib_event event; @@ -297,7 +297,7 @@ static struct ib_ah *hfi1_create_qp0_ah(struct hfi1_ibport *ibp, u32 dlid) struct rvt_qp *qp0; struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); struct hfi1_devdata *dd = dd_from_ppd(ppd); - u8 port_num = ppd->port; + u32 port_num = ppd->port; memset(&attr, 0, sizeof(attr)); attr.type = rdma_ah_find_type(&dd->verbs_dev.rdi.ibdev, port_num); @@ -515,7 +515,7 @@ static void bad_mkey(struct hfi1_ibport *ibp, struct ib_mad_hdr *mad, /* * Send a Port Capability Mask Changed trap (ch. 14.3.11). */ -void hfi1_cap_mask_chg(struct rvt_dev_info *rdi, u8 port_num) +void hfi1_cap_mask_chg(struct rvt_dev_info *rdi, u32 port_num) { struct trap_node *trap; struct hfi1_ibdev *verbs_dev = dev_from_rdi(rdi); @@ -581,7 +581,7 @@ void hfi1_node_desc_chg(struct hfi1_ibport *ibp) static int __subn_get_opa_nodedesc(struct opa_smp *smp, u32 am, u8 *data, struct ib_device *ibdev, - u8 port, u32 *resp_len, u32 max_len) + u32 port, u32 *resp_len, u32 max_len) { struct opa_node_description *nd; @@ -601,12 +601,12 @@ static int __subn_get_opa_nodedesc(struct opa_smp *smp, u32 am, } static int __subn_get_opa_nodeinfo(struct opa_smp *smp, u32 am, u8 *data, - struct ib_device *ibdev, u8 port, + struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len) { struct opa_node_info *ni; struct hfi1_devdata *dd = dd_from_ibdev(ibdev); - unsigned pidx = port - 1; /* IB number port from 1, hw from 0 */ + u32 pidx = port - 1; /* IB number port from 1, hw from 0 */ ni = (struct opa_node_info *)data; @@ -641,11 +641,11 @@ static int __subn_get_opa_nodeinfo(struct opa_smp *smp, u32 am, u8 *data, } static int subn_get_nodeinfo(struct ib_smp *smp, struct ib_device *ibdev, - u8 port) + u32 port) { struct ib_node_info *nip = (struct ib_node_info *)&smp->data; struct hfi1_devdata *dd = dd_from_ibdev(ibdev); - unsigned pidx = port - 1; /* IB number port from 1, hw from 0 */ + u32 pidx = port - 1; /* IB number port from 1, hw from 0 */ /* GUID 0 is illegal */ if (smp->attr_mod || pidx >= dd->num_pports || @@ -794,7 +794,7 @@ void read_ltp_rtt(struct hfi1_devdata *dd) } static int __subn_get_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data, - struct ib_device *ibdev, u8 port, + struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len) { int i; @@ -1009,7 +1009,7 @@ static int __subn_get_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data, * @port: the IB port number * @pkeys: the pkey table is placed here */ -static int get_pkeys(struct hfi1_devdata *dd, u8 port, u16 *pkeys) +static int get_pkeys(struct hfi1_devdata *dd, u32 port, u16 *pkeys) { struct hfi1_pportdata *ppd = dd->pport + port - 1; @@ -1019,7 +1019,7 @@ static int get_pkeys(struct hfi1_devdata *dd, u8 port, u16 *pkeys) } static int __subn_get_opa_pkeytable(struct opa_smp *smp, u32 am, u8 *data, - struct ib_device *ibdev, u8 port, + struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len) { struct hfi1_devdata *dd = dd_from_ibdev(ibdev); @@ -1349,7 +1349,7 @@ static int set_port_states(struct hfi1_pportdata *ppd, struct opa_smp *smp, * */ static int __subn_set_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data, - struct ib_device *ibdev, u8 port, + struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len, int local_mad) { struct opa_port_info *pi = (struct opa_port_info *)data; @@ -1667,7 +1667,7 @@ get_only: * @port: the IB port number * @pkeys: the PKEY table */ -static int set_pkeys(struct hfi1_devdata *dd, u8 port, u16 *pkeys) +static int set_pkeys(struct hfi1_devdata *dd, u32 port, u16 *pkeys) { struct hfi1_pportdata *ppd; int i; @@ -1718,7 +1718,7 @@ static int set_pkeys(struct hfi1_devdata *dd, u8 port, u16 *pkeys) } static int __subn_set_opa_pkeytable(struct opa_smp *smp, u32 am, u8 *data, - struct ib_device *ibdev, u8 port, + struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len) { struct hfi1_devdata *dd = dd_from_ibdev(ibdev); @@ -1732,7 +1732,7 @@ static int __subn_set_opa_pkeytable(struct opa_smp *smp, u32 am, u8 *data, u32 size = 0; if (n_blocks_sent == 0) { - pr_warn("OPA Get PKey AM Invalid : P = %d; B = 0x%x; N = 0x%x\n", + pr_warn("OPA Get PKey AM Invalid : P = %u; B = 0x%x; N = 0x%x\n", port, start_block, n_blocks_sent); smp->status |= IB_SMP_INVALID_FIELD; return reply((struct ib_mad_hdr *)smp); @@ -1825,7 +1825,7 @@ static int get_sc2vlt_tables(struct hfi1_devdata *dd, void *data) } static int __subn_get_opa_sl_to_sc(struct opa_smp *smp, u32 am, u8 *data, - struct ib_device *ibdev, u8 port, + struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len) { struct hfi1_ibport *ibp = to_iport(ibdev, port); @@ -1848,7 +1848,7 @@ static int __subn_get_opa_sl_to_sc(struct opa_smp *smp, u32 am, u8 *data, } static int __subn_set_opa_sl_to_sc(struct opa_smp *smp, u32 am, u8 *data, - struct ib_device *ibdev, u8 port, + struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len) { struct hfi1_ibport *ibp = to_iport(ibdev, port); @@ -1877,7 +1877,7 @@ static int __subn_set_opa_sl_to_sc(struct opa_smp *smp, u32 am, u8 *data, } static int __subn_get_opa_sc_to_sl(struct opa_smp *smp, u32 am, u8 *data, - struct ib_device *ibdev, u8 port, + struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len) { struct hfi1_ibport *ibp = to_iport(ibdev, port); @@ -1900,7 +1900,7 @@ static int __subn_get_opa_sc_to_sl(struct opa_smp *smp, u32 am, u8 *data, } static int __subn_set_opa_sc_to_sl(struct opa_smp *smp, u32 am, u8 *data, - struct ib_device *ibdev, u8 port, + struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len) { struct hfi1_ibport *ibp = to_iport(ibdev, port); @@ -1921,7 +1921,7 @@ static int __subn_set_opa_sc_to_sl(struct opa_smp *smp, u32 am, u8 *data, } static int __subn_get_opa_sc_to_vlt(struct opa_smp *smp, u32 am, u8 *data, - struct ib_device *ibdev, u8 port, + struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len) { u32 n_blocks = OPA_AM_NBLK(am); @@ -1943,7 +1943,7 @@ static int __subn_get_opa_sc_to_vlt(struct opa_smp *smp, u32 am, u8 *data, } static int __subn_set_opa_sc_to_vlt(struct opa_smp *smp, u32 am, u8 *data, - struct ib_device *ibdev, u8 port, + struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len) { u32 n_blocks = OPA_AM_NBLK(am); @@ -1985,7 +1985,7 @@ static int __subn_set_opa_sc_to_vlt(struct opa_smp *smp, u32 am, u8 *data, } static int __subn_get_opa_sc_to_vlnt(struct opa_smp *smp, u32 am, u8 *data, - struct ib_device *ibdev, u8 port, + struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len) { u32 n_blocks = OPA_AM_NPORT(am); @@ -2010,7 +2010,7 @@ static int __subn_get_opa_sc_to_vlnt(struct opa_smp *smp, u32 am, u8 *data, } static int __subn_set_opa_sc_to_vlnt(struct opa_smp *smp, u32 am, u8 *data, - struct ib_device *ibdev, u8 port, + struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len) { u32 n_blocks = OPA_AM_NPORT(am); @@ -2042,7 +2042,7 @@ static int __subn_set_opa_sc_to_vlnt(struct opa_smp *smp, u32 am, u8 *data, } static int __subn_get_opa_psi(struct opa_smp *smp, u32 am, u8 *data, - struct ib_device *ibdev, u8 port, + struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len) { u32 nports = OPA_AM_NPORT(am); @@ -2084,7 +2084,7 @@ static int __subn_get_opa_psi(struct opa_smp *smp, u32 am, u8 *data, } static int __subn_set_opa_psi(struct opa_smp *smp, u32 am, u8 *data, - struct ib_device *ibdev, u8 port, + struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len, int local_mad) { u32 nports = OPA_AM_NPORT(am); @@ -2132,7 +2132,7 @@ static int __subn_set_opa_psi(struct opa_smp *smp, u32 am, u8 *data, } static int __subn_get_opa_cable_info(struct opa_smp *smp, u32 am, u8 *data, - struct ib_device *ibdev, u8 port, + struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len) { struct hfi1_devdata *dd = dd_from_ibdev(ibdev); @@ -2184,7 +2184,7 @@ static int __subn_get_opa_cable_info(struct opa_smp *smp, u32 am, u8 *data, } static int __subn_get_opa_bct(struct opa_smp *smp, u32 am, u8 *data, - struct ib_device *ibdev, u8 port, u32 *resp_len, + struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len) { u32 num_ports = OPA_AM_NPORT(am); @@ -2208,7 +2208,7 @@ static int __subn_get_opa_bct(struct opa_smp *smp, u32 am, u8 *data, } static int __subn_set_opa_bct(struct opa_smp *smp, u32 am, u8 *data, - struct ib_device *ibdev, u8 port, u32 *resp_len, + struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len) { u32 num_ports = OPA_AM_NPORT(am); @@ -2232,7 +2232,7 @@ static int __subn_set_opa_bct(struct opa_smp *smp, u32 am, u8 *data, } static int __subn_get_opa_vl_arb(struct opa_smp *smp, u32 am, u8 *data, - struct ib_device *ibdev, u8 port, + struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len) { struct hfi1_pportdata *ppd = ppd_from_ibp(to_iport(ibdev, port)); @@ -2274,7 +2274,7 @@ static int __subn_get_opa_vl_arb(struct opa_smp *smp, u32 am, u8 *data, } static int __subn_set_opa_vl_arb(struct opa_smp *smp, u32 am, u8 *data, - struct ib_device *ibdev, u8 port, + struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len) { struct hfi1_pportdata *ppd = ppd_from_ibp(to_iport(ibdev, port)); @@ -2722,7 +2722,7 @@ u64 get_xmit_wait_counters(struct hfi1_pportdata *ppd, static int pma_get_opa_portstatus(struct opa_pma_mad *pmp, struct ib_device *ibdev, - u8 port, u32 *resp_len) + u32 port, u32 *resp_len) { struct opa_port_status_req *req = (struct opa_port_status_req *)pmp->data; @@ -2732,7 +2732,7 @@ static int pma_get_opa_portstatus(struct opa_pma_mad *pmp, unsigned long vl; size_t response_data_size; u32 nports = be32_to_cpu(pmp->mad_hdr.attr_mod) >> 24; - u8 port_num = req->port_num; + u32 port_num = req->port_num; u8 num_vls = hweight64(vl_select_mask); struct _vls_pctrs *vlinfo; struct hfi1_ibport *ibp = to_iport(ibdev, port); @@ -2888,7 +2888,7 @@ static int pma_get_opa_portstatus(struct opa_pma_mad *pmp, return reply((struct ib_mad_hdr *)pmp); } -static u64 get_error_counter_summary(struct ib_device *ibdev, u8 port, +static u64 get_error_counter_summary(struct ib_device *ibdev, u32 port, u8 res_lli, u8 res_ler) { struct hfi1_devdata *dd = dd_from_ibdev(ibdev); @@ -2973,7 +2973,7 @@ static void pma_get_opa_port_dctrs(struct ib_device *ibdev, static int pma_get_opa_datacounters(struct opa_pma_mad *pmp, struct ib_device *ibdev, - u8 port, u32 *resp_len) + u32 port, u32 *resp_len) { struct opa_port_data_counters_msg *req = (struct opa_port_data_counters_msg *)pmp->data; @@ -2987,7 +2987,7 @@ static int pma_get_opa_datacounters(struct opa_pma_mad *pmp, u8 lq, num_vls; u8 res_lli, res_ler; u64 port_mask; - u8 port_num; + u32 port_num; unsigned long vl; unsigned long vl_select_mask; int vfi; @@ -3123,7 +3123,7 @@ static int pma_get_opa_datacounters(struct opa_pma_mad *pmp, } static int pma_get_ib_portcounters_ext(struct ib_pma_mad *pmp, - struct ib_device *ibdev, u8 port) + struct ib_device *ibdev, u32 port) { struct ib_pma_portcounters_ext *p = (struct ib_pma_portcounters_ext *) pmp->data; @@ -3151,7 +3151,7 @@ bail: } static void pma_get_opa_port_ectrs(struct ib_device *ibdev, - struct _port_ectrs *rsp, u8 port) + struct _port_ectrs *rsp, u32 port) { u64 tmp, tmp2; struct hfi1_devdata *dd = dd_from_ibdev(ibdev); @@ -3194,11 +3194,11 @@ static void pma_get_opa_port_ectrs(struct ib_device *ibdev, static int pma_get_opa_porterrors(struct opa_pma_mad *pmp, struct ib_device *ibdev, - u8 port, u32 *resp_len) + u32 port, u32 *resp_len) { size_t response_data_size; struct _port_ectrs *rsp; - u8 port_num; + u32 port_num; struct opa_port_error_counters64_msg *req; struct hfi1_devdata *dd = dd_from_ibdev(ibdev); u32 num_ports; @@ -3283,7 +3283,7 @@ static int pma_get_opa_porterrors(struct opa_pma_mad *pmp, } static int pma_get_ib_portcounters(struct ib_pma_mad *pmp, - struct ib_device *ibdev, u8 port) + struct ib_device *ibdev, u32 port) { struct ib_pma_portcounters *p = (struct ib_pma_portcounters *) pmp->data; @@ -3369,7 +3369,7 @@ bail: static int pma_get_opa_errorinfo(struct opa_pma_mad *pmp, struct ib_device *ibdev, - u8 port, u32 *resp_len) + u32 port, u32 *resp_len) { size_t response_data_size; struct _port_ei *rsp; @@ -3377,7 +3377,7 @@ static int pma_get_opa_errorinfo(struct opa_pma_mad *pmp, struct hfi1_devdata *dd = dd_from_ibdev(ibdev); u64 port_mask; u32 num_ports; - u8 port_num; + u32 port_num; u8 num_pslm; u64 reg; @@ -3468,7 +3468,7 @@ static int pma_get_opa_errorinfo(struct opa_pma_mad *pmp, static int pma_set_opa_portstatus(struct opa_pma_mad *pmp, struct ib_device *ibdev, - u8 port, u32 *resp_len) + u32 port, u32 *resp_len) { struct opa_clear_port_status *req = (struct opa_clear_port_status *)pmp->data; @@ -3620,14 +3620,14 @@ static int pma_set_opa_portstatus(struct opa_pma_mad *pmp, static int pma_set_opa_errorinfo(struct opa_pma_mad *pmp, struct ib_device *ibdev, - u8 port, u32 *resp_len) + u32 port, u32 *resp_len) { struct _port_ei *rsp; struct opa_port_error_info_msg *req; struct hfi1_devdata *dd = dd_from_ibdev(ibdev); u64 port_mask; u32 num_ports; - u8 port_num; + u32 port_num; u8 num_pslm; u32 error_info_select; @@ -3702,7 +3702,7 @@ struct opa_congestion_info_attr { } __packed; static int __subn_get_opa_cong_info(struct opa_smp *smp, u32 am, u8 *data, - struct ib_device *ibdev, u8 port, + struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len) { struct opa_congestion_info_attr *p = @@ -3727,7 +3727,7 @@ static int __subn_get_opa_cong_info(struct opa_smp *smp, u32 am, u8 *data, static int __subn_get_opa_cong_setting(struct opa_smp *smp, u32 am, u8 *data, struct ib_device *ibdev, - u8 port, u32 *resp_len, u32 max_len) + u32 port, u32 *resp_len, u32 max_len) { int i; struct opa_congestion_setting_attr *p = @@ -3819,7 +3819,7 @@ static void apply_cc_state(struct hfi1_pportdata *ppd) } static int __subn_set_opa_cong_setting(struct opa_smp *smp, u32 am, u8 *data, - struct ib_device *ibdev, u8 port, + struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len) { struct opa_congestion_setting_attr *p = @@ -3860,7 +3860,7 @@ static int __subn_set_opa_cong_setting(struct opa_smp *smp, u32 am, u8 *data, static int __subn_get_opa_hfi1_cong_log(struct opa_smp *smp, u32 am, u8 *data, struct ib_device *ibdev, - u8 port, u32 *resp_len, u32 max_len) + u32 port, u32 *resp_len, u32 max_len) { struct hfi1_ibport *ibp = to_iport(ibdev, port); struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); @@ -3925,7 +3925,7 @@ static int __subn_get_opa_hfi1_cong_log(struct opa_smp *smp, u32 am, } static int __subn_get_opa_cc_table(struct opa_smp *smp, u32 am, u8 *data, - struct ib_device *ibdev, u8 port, + struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len) { struct ib_cc_table_attr *cc_table_attr = @@ -3977,7 +3977,7 @@ static int __subn_get_opa_cc_table(struct opa_smp *smp, u32 am, u8 *data, } static int __subn_set_opa_cc_table(struct opa_smp *smp, u32 am, u8 *data, - struct ib_device *ibdev, u8 port, + struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len) { struct ib_cc_table_attr *p = (struct ib_cc_table_attr *)data; @@ -4036,7 +4036,7 @@ struct opa_led_info { #define OPA_LED_MASK BIT(OPA_LED_SHIFT) static int __subn_get_opa_led_info(struct opa_smp *smp, u32 am, u8 *data, - struct ib_device *ibdev, u8 port, + struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len) { struct hfi1_devdata *dd = dd_from_ibdev(ibdev); @@ -4066,7 +4066,7 @@ static int __subn_get_opa_led_info(struct opa_smp *smp, u32 am, u8 *data, } static int __subn_set_opa_led_info(struct opa_smp *smp, u32 am, u8 *data, - struct ib_device *ibdev, u8 port, + struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len) { struct hfi1_devdata *dd = dd_from_ibdev(ibdev); @@ -4089,7 +4089,7 @@ static int __subn_set_opa_led_info(struct opa_smp *smp, u32 am, u8 *data, } static int subn_get_opa_sma(__be16 attr_id, struct opa_smp *smp, u32 am, - u8 *data, struct ib_device *ibdev, u8 port, + u8 *data, struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len) { int ret; @@ -4179,7 +4179,7 @@ static int subn_get_opa_sma(__be16 attr_id, struct opa_smp *smp, u32 am, } static int subn_set_opa_sma(__be16 attr_id, struct opa_smp *smp, u32 am, - u8 *data, struct ib_device *ibdev, u8 port, + u8 *data, struct ib_device *ibdev, u32 port, u32 *resp_len, u32 max_len, int local_mad) { int ret; @@ -4254,7 +4254,7 @@ static inline void set_aggr_error(struct opa_aggregate *ag) } static int subn_get_opa_aggregate(struct opa_smp *smp, - struct ib_device *ibdev, u8 port, + struct ib_device *ibdev, u32 port, u32 *resp_len) { int i; @@ -4303,7 +4303,7 @@ static int subn_get_opa_aggregate(struct opa_smp *smp, } static int subn_set_opa_aggregate(struct opa_smp *smp, - struct ib_device *ibdev, u8 port, + struct ib_device *ibdev, u32 port, u32 *resp_len, int local_mad) { int i; @@ -4509,7 +4509,7 @@ static int hfi1_pkey_validation_pma(struct hfi1_ibport *ibp, } static int process_subn_opa(struct ib_device *ibdev, int mad_flags, - u8 port, const struct opa_mad *in_mad, + u32 port, const struct opa_mad *in_mad, struct opa_mad *out_mad, u32 *resp_len, int local_mad) { @@ -4614,7 +4614,7 @@ static int process_subn_opa(struct ib_device *ibdev, int mad_flags, } static int process_subn(struct ib_device *ibdev, int mad_flags, - u8 port, const struct ib_mad *in_mad, + u32 port, const struct ib_mad *in_mad, struct ib_mad *out_mad) { struct ib_smp *smp = (struct ib_smp *)out_mad; @@ -4672,7 +4672,7 @@ static int process_subn(struct ib_device *ibdev, int mad_flags, return ret; } -static int process_perf(struct ib_device *ibdev, u8 port, +static int process_perf(struct ib_device *ibdev, u32 port, const struct ib_mad *in_mad, struct ib_mad *out_mad) { @@ -4734,7 +4734,7 @@ static int process_perf(struct ib_device *ibdev, u8 port, return ret; } -static int process_perf_opa(struct ib_device *ibdev, u8 port, +static int process_perf_opa(struct ib_device *ibdev, u32 port, const struct opa_mad *in_mad, struct opa_mad *out_mad, u32 *resp_len) { @@ -4816,7 +4816,7 @@ static int process_perf_opa(struct ib_device *ibdev, u8 port, } static int hfi1_process_opa_mad(struct ib_device *ibdev, int mad_flags, - u8 port, const struct ib_wc *in_wc, + u32 port, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const struct opa_mad *in_mad, struct opa_mad *out_mad, size_t *out_mad_size, @@ -4869,7 +4869,7 @@ bail: return ret; } -static int hfi1_process_ib_mad(struct ib_device *ibdev, int mad_flags, u8 port, +static int hfi1_process_ib_mad(struct ib_device *ibdev, int mad_flags, u32 port, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const struct ib_mad *in_mad, @@ -4914,7 +4914,7 @@ static int hfi1_process_ib_mad(struct ib_device *ibdev, int mad_flags, u8 port, * * This is called by the ib_mad module. */ -int hfi1_process_mad(struct ib_device *ibdev, int mad_flags, u8 port, +int hfi1_process_mad(struct ib_device *ibdev, int mad_flags, u32 port, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const struct ib_mad *in_mad, struct ib_mad *out_mad, size_t *out_mad_size, u16 *out_mad_pkey_index) diff --git a/drivers/infiniband/hw/hfi1/mad.h b/drivers/infiniband/hw/hfi1/mad.h index 889e63d3f2cc..0205d308ef5e 100644 --- a/drivers/infiniband/hw/hfi1/mad.h +++ b/drivers/infiniband/hw/hfi1/mad.h @@ -436,7 +436,7 @@ struct sc2vlnt { COUNTER_MASK(1, 3) | \ COUNTER_MASK(1, 4)) -void hfi1_event_pkey_change(struct hfi1_devdata *dd, u8 port); +void hfi1_event_pkey_change(struct hfi1_devdata *dd, u32 port); void hfi1_handle_trap_timer(struct timer_list *t); u16 tx_link_width(u16 link_width); u64 get_xmit_wait_counters(struct hfi1_pportdata *ppd, u16 link_width, diff --git a/drivers/infiniband/hw/hfi1/mmu_rb.c b/drivers/infiniband/hw/hfi1/mmu_rb.c index f3fb28e3d5d7..d213f65d4cdd 100644 --- a/drivers/infiniband/hw/hfi1/mmu_rb.c +++ b/drivers/infiniband/hw/hfi1/mmu_rb.c @@ -89,7 +89,7 @@ int hfi1_mmu_rb_register(void *ops_arg, struct mmu_rb_handler *h; int ret; - h = kmalloc(sizeof(*h), GFP_KERNEL); + h = kzalloc(sizeof(*h), GFP_KERNEL); if (!h) return -ENOMEM; diff --git a/drivers/infiniband/hw/hfi1/msix.c b/drivers/infiniband/hw/hfi1/msix.c index cf3040bb177f..57a5f02ebc77 100644 --- a/drivers/infiniband/hw/hfi1/msix.c +++ b/drivers/infiniband/hw/hfi1/msix.c @@ -206,7 +206,7 @@ int msix_request_rcd_irq(struct hfi1_ctxtdata *rcd) } /** - * msix_request_rcd_irq() - Helper function for RCVAVAIL IRQs + * msix_netdev_request_rcd_irq - Helper function for RCVAVAIL IRQs * for netdev context * @rcd: valid netdev contexti */ @@ -221,7 +221,7 @@ int msix_netdev_request_rcd_irq(struct hfi1_ctxtdata *rcd) } /** - * msix_request_smda_ira() - Helper for getting SDMA IRQ resources + * msix_request_sdma_irq - Helper for getting SDMA IRQ resources * @sde: valid sdma engine * */ @@ -243,7 +243,7 @@ int msix_request_sdma_irq(struct sdma_engine *sde) } /** - * msix_request_general_irq(void) - Helper for getting general IRQ + * msix_request_general_irq - Helper for getting general IRQ * resources * @dd: valid device data */ @@ -269,7 +269,7 @@ int msix_request_general_irq(struct hfi1_devdata *dd) } /** - * enable_sdma_src() - Helper to enable SDMA IRQ srcs + * enable_sdma_srcs - Helper to enable SDMA IRQ srcs * @dd: valid devdata structure * @i: index of SDMA engine */ @@ -349,7 +349,7 @@ void msix_free_irq(struct hfi1_devdata *dd, u8 msix_intr) } /** - * hfi1_clean_up_msix_interrupts() - Free all MSIx IRQ resources + * msix_clean_up_interrupts - Free all MSIx IRQ resources * @dd: valid device data data structure * * Free the MSIx and associated PCI resources, if they have been allocated. @@ -372,7 +372,7 @@ void msix_clean_up_interrupts(struct hfi1_devdata *dd) } /** - * msix_netdev_syncrhonize_irq() - netdev IRQ synchronize + * msix_netdev_synchronize_irq - netdev IRQ synchronize * @dd: valid devdata */ void msix_netdev_synchronize_irq(struct hfi1_devdata *dd) diff --git a/drivers/infiniband/hw/hfi1/netdev.h b/drivers/infiniband/hw/hfi1/netdev.h index 947543a3e0c4..8aa074670a9c 100644 --- a/drivers/infiniband/hw/hfi1/netdev.h +++ b/drivers/infiniband/hw/hfi1/netdev.h @@ -14,15 +14,14 @@ /** * struct hfi1_netdev_rxq - Receive Queue for HFI - * dummy netdev. Both IPoIB and VNIC netdevices will be working on - * top of this device. + * Both IPoIB and VNIC netdevices will be working on the rx abstraction. * @napi: napi object - * @priv: ptr to netdev_priv + * @rx: ptr to netdev_rx * @rcd: ptr to receive context data */ struct hfi1_netdev_rxq { struct napi_struct napi; - struct hfi1_netdev_priv *priv; + struct hfi1_netdev_rx *rx; struct hfi1_ctxtdata *rcd; }; @@ -36,7 +35,8 @@ struct hfi1_netdev_rxq { #define NUM_NETDEV_MAP_ENTRIES HFI1_MAX_NETDEV_CTXTS /** - * struct hfi1_netdev_priv: data required to setup and run HFI netdev. + * struct hfi1_netdev_rx: data required to setup and run HFI netdev. + * @rx_napi: the dummy netdevice to support "polling" the receive contexts * @dd: hfi1_devdata * @rxq: pointer to dummy netdev receive queues. * @num_rx_q: number of receive queues @@ -48,7 +48,8 @@ struct hfi1_netdev_rxq { * @netdevs: atomic counter of netdevs using dummy netdev. * When 0 receive queues will be freed. */ -struct hfi1_netdev_priv { +struct hfi1_netdev_rx { + struct net_device rx_napi; struct hfi1_devdata *dd; struct hfi1_netdev_rxq *rxq; int num_rx_q; @@ -61,41 +62,27 @@ struct hfi1_netdev_priv { }; static inline -struct hfi1_netdev_priv *hfi1_netdev_priv(struct net_device *dev) -{ - return (struct hfi1_netdev_priv *)&dev[1]; -} - -static inline int hfi1_netdev_ctxt_count(struct hfi1_devdata *dd) { - struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev); - - return priv->num_rx_q; + return dd->netdev_rx->num_rx_q; } static inline struct hfi1_ctxtdata *hfi1_netdev_get_ctxt(struct hfi1_devdata *dd, int ctxt) { - struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev); - - return priv->rxq[ctxt].rcd; + return dd->netdev_rx->rxq[ctxt].rcd; } static inline int hfi1_netdev_get_free_rmt_idx(struct hfi1_devdata *dd) { - struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev); - - return priv->rmt_start; + return dd->netdev_rx->rmt_start; } static inline void hfi1_netdev_set_free_rmt_idx(struct hfi1_devdata *dd, int rmt_idx) { - struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev); - - priv->rmt_start = rmt_idx; + dd->netdev_rx->rmt_start = rmt_idx; } u32 hfi1_num_netdev_contexts(struct hfi1_devdata *dd, u32 available_contexts, @@ -105,8 +92,8 @@ void hfi1_netdev_enable_queues(struct hfi1_devdata *dd); void hfi1_netdev_disable_queues(struct hfi1_devdata *dd); int hfi1_netdev_rx_init(struct hfi1_devdata *dd); int hfi1_netdev_rx_destroy(struct hfi1_devdata *dd); -int hfi1_netdev_alloc(struct hfi1_devdata *dd); -void hfi1_netdev_free(struct hfi1_devdata *dd); +int hfi1_alloc_rx(struct hfi1_devdata *dd); +void hfi1_free_rx(struct hfi1_devdata *dd); int hfi1_netdev_add_data(struct hfi1_devdata *dd, int id, void *data); void *hfi1_netdev_remove_data(struct hfi1_devdata *dd, int id); void *hfi1_netdev_get_data(struct hfi1_devdata *dd, int id); diff --git a/drivers/infiniband/hw/hfi1/netdev_rx.c b/drivers/infiniband/hw/hfi1/netdev_rx.c index 1bcab992ac26..03b098a494b5 100644 --- a/drivers/infiniband/hw/hfi1/netdev_rx.c +++ b/drivers/infiniband/hw/hfi1/netdev_rx.c @@ -17,11 +17,11 @@ #include <linux/etherdevice.h> #include <rdma/ib_verbs.h> -static int hfi1_netdev_setup_ctxt(struct hfi1_netdev_priv *priv, +static int hfi1_netdev_setup_ctxt(struct hfi1_netdev_rx *rx, struct hfi1_ctxtdata *uctxt) { unsigned int rcvctrl_ops; - struct hfi1_devdata *dd = priv->dd; + struct hfi1_devdata *dd = rx->dd; int ret; uctxt->rhf_rcv_function_map = netdev_rhf_rcv_functions; @@ -118,11 +118,11 @@ static void hfi1_netdev_deallocate_ctxt(struct hfi1_devdata *dd, hfi1_free_ctxt(uctxt); } -static int hfi1_netdev_allot_ctxt(struct hfi1_netdev_priv *priv, +static int hfi1_netdev_allot_ctxt(struct hfi1_netdev_rx *rx, struct hfi1_ctxtdata **ctxt) { int rc; - struct hfi1_devdata *dd = priv->dd; + struct hfi1_devdata *dd = rx->dd; rc = hfi1_netdev_allocate_ctxt(dd, ctxt); if (rc) { @@ -130,7 +130,7 @@ static int hfi1_netdev_allot_ctxt(struct hfi1_netdev_priv *priv, return rc; } - rc = hfi1_netdev_setup_ctxt(priv, *ctxt); + rc = hfi1_netdev_setup_ctxt(rx, *ctxt); if (rc) { dd_dev_err(dd, "netdev ctxt setup failed %d\n", rc); hfi1_netdev_deallocate_ctxt(dd, *ctxt); @@ -183,31 +183,31 @@ u32 hfi1_num_netdev_contexts(struct hfi1_devdata *dd, u32 available_contexts, (u32)HFI1_MAX_NETDEV_CTXTS); } -static int hfi1_netdev_rxq_init(struct net_device *dev) +static int hfi1_netdev_rxq_init(struct hfi1_netdev_rx *rx) { int i; int rc; - struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dev); - struct hfi1_devdata *dd = priv->dd; + struct hfi1_devdata *dd = rx->dd; + struct net_device *dev = &rx->rx_napi; - priv->num_rx_q = dd->num_netdev_contexts; - priv->rxq = kcalloc_node(priv->num_rx_q, sizeof(struct hfi1_netdev_rxq), - GFP_KERNEL, dd->node); + rx->num_rx_q = dd->num_netdev_contexts; + rx->rxq = kcalloc_node(rx->num_rx_q, sizeof(*rx->rxq), + GFP_KERNEL, dd->node); - if (!priv->rxq) { + if (!rx->rxq) { dd_dev_err(dd, "Unable to allocate netdev queue data\n"); return (-ENOMEM); } - for (i = 0; i < priv->num_rx_q; i++) { - struct hfi1_netdev_rxq *rxq = &priv->rxq[i]; + for (i = 0; i < rx->num_rx_q; i++) { + struct hfi1_netdev_rxq *rxq = &rx->rxq[i]; - rc = hfi1_netdev_allot_ctxt(priv, &rxq->rcd); + rc = hfi1_netdev_allot_ctxt(rx, &rxq->rcd); if (rc) goto bail_context_irq_failure; hfi1_rcd_get(rxq->rcd); - rxq->priv = priv; + rxq->rx = rx; rxq->rcd->napi = &rxq->napi; dd_dev_info(dd, "Setting rcv queue %d napi to context %d\n", i, rxq->rcd->ctxt); @@ -227,7 +227,7 @@ static int hfi1_netdev_rxq_init(struct net_device *dev) bail_context_irq_failure: dd_dev_err(dd, "Unable to allot receive context\n"); for (; i >= 0; i--) { - struct hfi1_netdev_rxq *rxq = &priv->rxq[i]; + struct hfi1_netdev_rxq *rxq = &rx->rxq[i]; if (rxq->rcd) { hfi1_netdev_deallocate_ctxt(dd, rxq->rcd); @@ -235,20 +235,19 @@ bail_context_irq_failure: rxq->rcd = NULL; } } - kfree(priv->rxq); - priv->rxq = NULL; + kfree(rx->rxq); + rx->rxq = NULL; return rc; } -static void hfi1_netdev_rxq_deinit(struct net_device *dev) +static void hfi1_netdev_rxq_deinit(struct hfi1_netdev_rx *rx) { int i; - struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dev); - struct hfi1_devdata *dd = priv->dd; + struct hfi1_devdata *dd = rx->dd; - for (i = 0; i < priv->num_rx_q; i++) { - struct hfi1_netdev_rxq *rxq = &priv->rxq[i]; + for (i = 0; i < rx->num_rx_q; i++) { + struct hfi1_netdev_rxq *rxq = &rx->rxq[i]; netif_napi_del(&rxq->napi); hfi1_netdev_deallocate_ctxt(dd, rxq->rcd); @@ -256,41 +255,41 @@ static void hfi1_netdev_rxq_deinit(struct net_device *dev) rxq->rcd = NULL; } - kfree(priv->rxq); - priv->rxq = NULL; - priv->num_rx_q = 0; + kfree(rx->rxq); + rx->rxq = NULL; + rx->num_rx_q = 0; } -static void enable_queues(struct hfi1_netdev_priv *priv) +static void enable_queues(struct hfi1_netdev_rx *rx) { int i; - for (i = 0; i < priv->num_rx_q; i++) { - struct hfi1_netdev_rxq *rxq = &priv->rxq[i]; + for (i = 0; i < rx->num_rx_q; i++) { + struct hfi1_netdev_rxq *rxq = &rx->rxq[i]; - dd_dev_info(priv->dd, "enabling queue %d on context %d\n", i, + dd_dev_info(rx->dd, "enabling queue %d on context %d\n", i, rxq->rcd->ctxt); napi_enable(&rxq->napi); - hfi1_rcvctrl(priv->dd, + hfi1_rcvctrl(rx->dd, HFI1_RCVCTRL_CTXT_ENB | HFI1_RCVCTRL_INTRAVAIL_ENB, rxq->rcd); } } -static void disable_queues(struct hfi1_netdev_priv *priv) +static void disable_queues(struct hfi1_netdev_rx *rx) { int i; - msix_netdev_synchronize_irq(priv->dd); + msix_netdev_synchronize_irq(rx->dd); - for (i = 0; i < priv->num_rx_q; i++) { - struct hfi1_netdev_rxq *rxq = &priv->rxq[i]; + for (i = 0; i < rx->num_rx_q; i++) { + struct hfi1_netdev_rxq *rxq = &rx->rxq[i]; - dd_dev_info(priv->dd, "disabling queue %d on context %d\n", i, + dd_dev_info(rx->dd, "disabling queue %d on context %d\n", i, rxq->rcd->ctxt); /* wait for napi if it was scheduled */ - hfi1_rcvctrl(priv->dd, + hfi1_rcvctrl(rx->dd, HFI1_RCVCTRL_CTXT_DIS | HFI1_RCVCTRL_INTRAVAIL_DIS, rxq->rcd); napi_synchronize(&rxq->napi); @@ -307,15 +306,14 @@ static void disable_queues(struct hfi1_netdev_priv *priv) */ int hfi1_netdev_rx_init(struct hfi1_devdata *dd) { - struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev); + struct hfi1_netdev_rx *rx = dd->netdev_rx; int res; - if (atomic_fetch_inc(&priv->netdevs)) + if (atomic_fetch_inc(&rx->netdevs)) return 0; mutex_lock(&hfi1_mutex); - init_dummy_netdev(dd->dummy_netdev); - res = hfi1_netdev_rxq_init(dd->dummy_netdev); + res = hfi1_netdev_rxq_init(rx); mutex_unlock(&hfi1_mutex); return res; } @@ -328,12 +326,12 @@ int hfi1_netdev_rx_init(struct hfi1_devdata *dd) */ int hfi1_netdev_rx_destroy(struct hfi1_devdata *dd) { - struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev); + struct hfi1_netdev_rx *rx = dd->netdev_rx; /* destroy the RX queues only if it is the last netdev going away */ - if (atomic_fetch_add_unless(&priv->netdevs, -1, 0) == 1) { + if (atomic_fetch_add_unless(&rx->netdevs, -1, 0) == 1) { mutex_lock(&hfi1_mutex); - hfi1_netdev_rxq_deinit(dd->dummy_netdev); + hfi1_netdev_rxq_deinit(rx); mutex_unlock(&hfi1_mutex); } @@ -341,39 +339,43 @@ int hfi1_netdev_rx_destroy(struct hfi1_devdata *dd) } /** - * hfi1_netdev_alloc - Allocates netdev and private data. It is required - * because RMT index and MSI-X interrupt can be set only - * during driver initialization. - * + * hfi1_alloc_rx - Allocates the rx support structure * @dd: hfi1 dev data + * + * Allocate the rx structure to support gathering the receive + * resources and the dummy netdev. + * + * Updates dd struct pointer upon success. + * + * Return: 0 (success) -error on failure + * */ -int hfi1_netdev_alloc(struct hfi1_devdata *dd) +int hfi1_alloc_rx(struct hfi1_devdata *dd) { - struct hfi1_netdev_priv *priv; - const int netdev_size = sizeof(*dd->dummy_netdev) + - sizeof(struct hfi1_netdev_priv); + struct hfi1_netdev_rx *rx; - dd_dev_info(dd, "allocating netdev size %d\n", netdev_size); - dd->dummy_netdev = kcalloc_node(1, netdev_size, GFP_KERNEL, dd->node); + dd_dev_info(dd, "allocating rx size %ld\n", sizeof(*rx)); + rx = kzalloc_node(sizeof(*rx), GFP_KERNEL, dd->node); - if (!dd->dummy_netdev) + if (!rx) return -ENOMEM; + rx->dd = dd; + init_dummy_netdev(&rx->rx_napi); - priv = hfi1_netdev_priv(dd->dummy_netdev); - priv->dd = dd; - xa_init(&priv->dev_tbl); - atomic_set(&priv->enabled, 0); - atomic_set(&priv->netdevs, 0); + xa_init(&rx->dev_tbl); + atomic_set(&rx->enabled, 0); + atomic_set(&rx->netdevs, 0); + dd->netdev_rx = rx; return 0; } -void hfi1_netdev_free(struct hfi1_devdata *dd) +void hfi1_free_rx(struct hfi1_devdata *dd) { - if (dd->dummy_netdev) { - dd_dev_info(dd, "hfi1 netdev freed\n"); - kfree(dd->dummy_netdev); - dd->dummy_netdev = NULL; + if (dd->netdev_rx) { + dd_dev_info(dd, "hfi1 rx freed\n"); + kfree(dd->netdev_rx); + dd->netdev_rx = NULL; } } @@ -388,33 +390,33 @@ void hfi1_netdev_free(struct hfi1_devdata *dd) */ void hfi1_netdev_enable_queues(struct hfi1_devdata *dd) { - struct hfi1_netdev_priv *priv; + struct hfi1_netdev_rx *rx; - if (!dd->dummy_netdev) + if (!dd->netdev_rx) return; - priv = hfi1_netdev_priv(dd->dummy_netdev); - if (atomic_fetch_inc(&priv->enabled)) + rx = dd->netdev_rx; + if (atomic_fetch_inc(&rx->enabled)) return; mutex_lock(&hfi1_mutex); - enable_queues(priv); + enable_queues(rx); mutex_unlock(&hfi1_mutex); } void hfi1_netdev_disable_queues(struct hfi1_devdata *dd) { - struct hfi1_netdev_priv *priv; + struct hfi1_netdev_rx *rx; - if (!dd->dummy_netdev) + if (!dd->netdev_rx) return; - priv = hfi1_netdev_priv(dd->dummy_netdev); - if (atomic_dec_if_positive(&priv->enabled)) + rx = dd->netdev_rx; + if (atomic_dec_if_positive(&rx->enabled)) return; mutex_lock(&hfi1_mutex); - disable_queues(priv); + disable_queues(rx); mutex_unlock(&hfi1_mutex); } @@ -430,9 +432,9 @@ void hfi1_netdev_disable_queues(struct hfi1_devdata *dd) */ int hfi1_netdev_add_data(struct hfi1_devdata *dd, int id, void *data) { - struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev); + struct hfi1_netdev_rx *rx = dd->netdev_rx; - return xa_insert(&priv->dev_tbl, id, data, GFP_NOWAIT); + return xa_insert(&rx->dev_tbl, id, data, GFP_NOWAIT); } /** @@ -444,9 +446,9 @@ int hfi1_netdev_add_data(struct hfi1_devdata *dd, int id, void *data) */ void *hfi1_netdev_remove_data(struct hfi1_devdata *dd, int id) { - struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev); + struct hfi1_netdev_rx *rx = dd->netdev_rx; - return xa_erase(&priv->dev_tbl, id); + return xa_erase(&rx->dev_tbl, id); } /** @@ -457,24 +459,24 @@ void *hfi1_netdev_remove_data(struct hfi1_devdata *dd, int id) */ void *hfi1_netdev_get_data(struct hfi1_devdata *dd, int id) { - struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev); + struct hfi1_netdev_rx *rx = dd->netdev_rx; - return xa_load(&priv->dev_tbl, id); + return xa_load(&rx->dev_tbl, id); } /** - * hfi1_netdev_get_first_dat - Gets first entry with greater or equal id. + * hfi1_netdev_get_first_data - Gets first entry with greater or equal id. * * @dd: hfi1 dev data * @start_id: requested integer id up to INT_MAX */ void *hfi1_netdev_get_first_data(struct hfi1_devdata *dd, int *start_id) { - struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev); + struct hfi1_netdev_rx *rx = dd->netdev_rx; unsigned long index = *start_id; void *ret; - ret = xa_find(&priv->dev_tbl, &index, UINT_MAX, XA_PRESENT); + ret = xa_find(&rx->dev_tbl, &index, UINT_MAX, XA_PRESENT); *start_id = (int)index; return ret; } diff --git a/drivers/infiniband/hw/hfi1/sdma.c b/drivers/infiniband/hw/hfi1/sdma.c index 46b5290b2839..1fcc6e9666e0 100644 --- a/drivers/infiniband/hw/hfi1/sdma.c +++ b/drivers/infiniband/hw/hfi1/sdma.c @@ -1285,7 +1285,7 @@ bail: } /** - * sdma_clean() Clean up allocated memory + * sdma_clean - Clean up allocated memory * @dd: struct hfi1_devdata * @num_engines: num sdma engines * diff --git a/drivers/infiniband/hw/hfi1/sdma.h b/drivers/infiniband/hw/hfi1/sdma.h index 7a851191f987..f57d55272dd2 100644 --- a/drivers/infiniband/hw/hfi1/sdma.h +++ b/drivers/infiniband/hw/hfi1/sdma.h @@ -907,24 +907,6 @@ static inline unsigned sdma_progress(struct sdma_engine *sde, unsigned seq, return 0; } -/** - * sdma_iowait_schedule() - initialize wait structure - * @sde: sdma_engine to schedule - * @wait: wait struct to schedule - * - * This function initializes the iowait - * structure embedded in the QP or PQ. - * - */ -static inline void sdma_iowait_schedule( - struct sdma_engine *sde, - struct iowait *wait) -{ - struct hfi1_pportdata *ppd = sde->dd->pport; - - iowait_schedule(wait, ppd->hfi1_wq, sde->cpu); -} - /* for use by interrupt handling */ void sdma_engine_error(struct sdma_engine *sde, u64 status); void sdma_engine_interrupt(struct sdma_engine *sde, u64 status); diff --git a/drivers/infiniband/hw/hfi1/sysfs.c b/drivers/infiniband/hw/hfi1/sysfs.c index 5650130e68d4..eaf441ece25e 100644 --- a/drivers/infiniband/hw/hfi1/sysfs.c +++ b/drivers/infiniband/hw/hfi1/sysfs.c @@ -649,7 +649,7 @@ const struct attribute_group ib_hfi1_attr_group = { .attrs = hfi1_attributes, }; -int hfi1_create_port_files(struct ib_device *ibdev, u8 port_num, +int hfi1_create_port_files(struct ib_device *ibdev, u32 port_num, struct kobject *kobj) { struct hfi1_pportdata *ppd; diff --git a/drivers/infiniband/hw/hfi1/trace_tx.h b/drivers/infiniband/hw/hfi1/trace_tx.h index 769e5e4710c6..d44fc54858b9 100644 --- a/drivers/infiniband/hw/hfi1/trace_tx.h +++ b/drivers/infiniband/hw/hfi1/trace_tx.h @@ -53,6 +53,8 @@ #include "hfi.h" #include "mad.h" #include "sdma.h" +#include "ipoib.h" +#include "user_sdma.h" const char *parse_sdma_flags(struct trace_seq *p, u64 desc0, u64 desc1); @@ -653,6 +655,80 @@ TRACE_EVENT(hfi1_sdma_user_completion, __entry->code) ); +TRACE_EVENT(hfi1_usdma_defer, + TP_PROTO(struct hfi1_user_sdma_pkt_q *pq, + struct sdma_engine *sde, + struct iowait *wait), + TP_ARGS(pq, sde, wait), + TP_STRUCT__entry(DD_DEV_ENTRY(pq->dd) + __field(struct hfi1_user_sdma_pkt_q *, pq) + __field(struct sdma_engine *, sde) + __field(struct iowait *, wait) + __field(int, engine) + __field(int, empty) + ), + TP_fast_assign(DD_DEV_ASSIGN(pq->dd); + __entry->pq = pq; + __entry->sde = sde; + __entry->wait = wait; + __entry->engine = sde->this_idx; + __entry->empty = list_empty(&__entry->wait->list); + ), + TP_printk("[%s] pq %llx sde %llx wait %llx engine %d empty %d", + __get_str(dev), + (unsigned long long)__entry->pq, + (unsigned long long)__entry->sde, + (unsigned long long)__entry->wait, + __entry->engine, + __entry->empty + ) +); + +TRACE_EVENT(hfi1_usdma_activate, + TP_PROTO(struct hfi1_user_sdma_pkt_q *pq, + struct iowait *wait, + int reason), + TP_ARGS(pq, wait, reason), + TP_STRUCT__entry(DD_DEV_ENTRY(pq->dd) + __field(struct hfi1_user_sdma_pkt_q *, pq) + __field(struct iowait *, wait) + __field(int, reason) + ), + TP_fast_assign(DD_DEV_ASSIGN(pq->dd); + __entry->pq = pq; + __entry->wait = wait; + __entry->reason = reason; + ), + TP_printk("[%s] pq %llx wait %llx reason %d", + __get_str(dev), + (unsigned long long)__entry->pq, + (unsigned long long)__entry->wait, + __entry->reason + ) +); + +TRACE_EVENT(hfi1_usdma_we, + TP_PROTO(struct hfi1_user_sdma_pkt_q *pq, + int we_ret), + TP_ARGS(pq, we_ret), + TP_STRUCT__entry(DD_DEV_ENTRY(pq->dd) + __field(struct hfi1_user_sdma_pkt_q *, pq) + __field(int, state) + __field(int, we_ret) + ), + TP_fast_assign(DD_DEV_ASSIGN(pq->dd); + __entry->pq = pq; + __entry->state = pq->state; + __entry->we_ret = we_ret; + ), + TP_printk("[%s] pq %llx state %d we_ret %d", + __get_str(dev), + (unsigned long long)__entry->pq, + __entry->state, + __entry->we_ret + ) +); + const char *print_u32_array(struct trace_seq *, u32 *, int); #define __print_u32_hex(arr, len) print_u32_array(p, arr, len) @@ -858,6 +934,109 @@ DEFINE_EVENT( TP_ARGS(qp, flag) ); +DECLARE_EVENT_CLASS(/* AIP */ + hfi1_ipoib_txq_template, + TP_PROTO(struct hfi1_ipoib_txq *txq), + TP_ARGS(txq), + TP_STRUCT__entry(/* entry */ + DD_DEV_ENTRY(txq->priv->dd) + __field(struct hfi1_ipoib_txq *, txq) + __field(struct sdma_engine *, sde) + __field(ulong, head) + __field(ulong, tail) + __field(uint, used) + __field(uint, flow) + __field(int, stops) + __field(int, no_desc) + __field(u8, idx) + __field(u8, stopped) + ), + TP_fast_assign(/* assign */ + DD_DEV_ASSIGN(txq->priv->dd) + __entry->txq = txq; + __entry->sde = txq->sde; + __entry->head = txq->tx_ring.head; + __entry->tail = txq->tx_ring.tail; + __entry->idx = txq->q_idx; + __entry->used = + txq->sent_txreqs - + atomic64_read(&txq->complete_txreqs); + __entry->flow = txq->flow.as_int; + __entry->stops = atomic_read(&txq->stops); + __entry->no_desc = atomic_read(&txq->no_desc); + __entry->stopped = + __netif_subqueue_stopped(txq->priv->netdev, txq->q_idx); + ), + TP_printk(/* print */ + "[%s] txq %llx idx %u sde %llx head %lx tail %lx flow %x used %u stops %d no_desc %d stopped %u", + __get_str(dev), + (unsigned long long)__entry->txq, + __entry->idx, + (unsigned long long)__entry->sde, + __entry->head, + __entry->tail, + __entry->flow, + __entry->used, + __entry->stops, + __entry->no_desc, + __entry->stopped + ) +); + +DEFINE_EVENT(/* queue stop */ + hfi1_ipoib_txq_template, hfi1_txq_stop, + TP_PROTO(struct hfi1_ipoib_txq *txq), + TP_ARGS(txq) +); + +DEFINE_EVENT(/* queue wake */ + hfi1_ipoib_txq_template, hfi1_txq_wake, + TP_PROTO(struct hfi1_ipoib_txq *txq), + TP_ARGS(txq) +); + +DEFINE_EVENT(/* flow flush */ + hfi1_ipoib_txq_template, hfi1_flow_flush, + TP_PROTO(struct hfi1_ipoib_txq *txq), + TP_ARGS(txq) +); + +DEFINE_EVENT(/* flow switch */ + hfi1_ipoib_txq_template, hfi1_flow_switch, + TP_PROTO(struct hfi1_ipoib_txq *txq), + TP_ARGS(txq) +); + +DEFINE_EVENT(/* wakeup */ + hfi1_ipoib_txq_template, hfi1_txq_wakeup, + TP_PROTO(struct hfi1_ipoib_txq *txq), + TP_ARGS(txq) +); + +DEFINE_EVENT(/* full */ + hfi1_ipoib_txq_template, hfi1_txq_full, + TP_PROTO(struct hfi1_ipoib_txq *txq), + TP_ARGS(txq) +); + +DEFINE_EVENT(/* queued */ + hfi1_ipoib_txq_template, hfi1_txq_queued, + TP_PROTO(struct hfi1_ipoib_txq *txq), + TP_ARGS(txq) +); + +DEFINE_EVENT(/* xmit_stopped */ + hfi1_ipoib_txq_template, hfi1_txq_xmit_stopped, + TP_PROTO(struct hfi1_ipoib_txq *txq), + TP_ARGS(txq) +); + +DEFINE_EVENT(/* xmit_unstopped */ + hfi1_ipoib_txq_template, hfi1_txq_xmit_unstopped, + TP_PROTO(struct hfi1_ipoib_txq *txq), + TP_ARGS(txq) +); + #endif /* __HFI1_TRACE_TX_H */ #undef TRACE_INCLUDE_PATH diff --git a/drivers/infiniband/hw/hfi1/user_sdma.c b/drivers/infiniband/hw/hfi1/user_sdma.c index 4a4956f96a7e..da5b2e37355a 100644 --- a/drivers/infiniband/hw/hfi1/user_sdma.c +++ b/drivers/infiniband/hw/hfi1/user_sdma.c @@ -133,6 +133,7 @@ static int defer_packet_queue( container_of(wait->iow, struct hfi1_user_sdma_pkt_q, busy); write_seqlock(&sde->waitlock); + trace_hfi1_usdma_defer(pq, sde, &pq->busy); if (sdma_progress(sde, seq, txreq)) goto eagain; /* @@ -157,7 +158,8 @@ static void activate_packet_queue(struct iowait *wait, int reason) { struct hfi1_user_sdma_pkt_q *pq = container_of(wait, struct hfi1_user_sdma_pkt_q, busy); - pq->busy.lock = NULL; + + trace_hfi1_usdma_activate(pq, wait, reason); xchg(&pq->state, SDMA_PKT_Q_ACTIVE); wake_up(&wait->wait_dma); }; @@ -599,13 +601,17 @@ int hfi1_user_sdma_process_request(struct hfi1_filedata *fd, while (req->seqsubmitted != req->info.npkts) { ret = user_sdma_send_pkts(req, pcount); if (ret < 0) { + int we_ret; + if (ret != -EBUSY) goto free_req; - if (wait_event_interruptible_timeout( + we_ret = wait_event_interruptible_timeout( pq->busy.wait_dma, pq->state == SDMA_PKT_Q_ACTIVE, msecs_to_jiffies( - SDMA_IOWAIT_TIMEOUT)) <= 0) + SDMA_IOWAIT_TIMEOUT)); + trace_hfi1_usdma_we(pq, we_ret); + if (we_ret <= 0) flush_pq_iowait(pq); } } diff --git a/drivers/infiniband/hw/hfi1/user_sdma.h b/drivers/infiniband/hw/hfi1/user_sdma.h index 1e8c02fe8ad1..fabe58139906 100644 --- a/drivers/infiniband/hw/hfi1/user_sdma.h +++ b/drivers/infiniband/hw/hfi1/user_sdma.h @@ -53,6 +53,7 @@ #include "common.h" #include "iowait.h" #include "user_exp_rcv.h" +#include "mmu_rb.h" /* The maximum number of Data io vectors per message/request */ #define MAX_VECTORS_PER_REQ 8 diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c index 0dd4bb0a5a7e..554294340caa 100644 --- a/drivers/infiniband/hw/hfi1/verbs.c +++ b/drivers/infiniband/hw/hfi1/verbs.c @@ -1407,7 +1407,7 @@ static inline u16 opa_width_to_ib(u16 in) } } -static int query_port(struct rvt_dev_info *rdi, u8 port_num, +static int query_port(struct rvt_dev_info *rdi, u32 port_num, struct ib_port_attr *props) { struct hfi1_ibdev *verbs_dev = dev_from_rdi(rdi); @@ -1485,7 +1485,7 @@ bail: return ret; } -static int shut_down_port(struct rvt_dev_info *rdi, u8 port_num) +static int shut_down_port(struct rvt_dev_info *rdi, u32 port_num) { struct hfi1_ibdev *verbs_dev = dev_from_rdi(rdi); struct hfi1_devdata *dd = dd_from_dev(verbs_dev); @@ -1694,7 +1694,7 @@ static int init_cntr_names(const char *names_in, } static struct rdma_hw_stats *alloc_hw_stats(struct ib_device *ibdev, - u8 port_num) + u32 port_num) { int i, err; @@ -1758,7 +1758,7 @@ static u64 hfi1_sps_ints(void) } static int get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, - u8 port, int index) + u32 port, int index) { u64 *values; int count; diff --git a/drivers/infiniband/hw/hfi1/verbs.h b/drivers/infiniband/hw/hfi1/verbs.h index d36e3e14896d..420df17cd184 100644 --- a/drivers/infiniband/hw/hfi1/verbs.h +++ b/drivers/infiniband/hw/hfi1/verbs.h @@ -325,10 +325,10 @@ static inline struct rvt_qp *iowait_to_qp(struct iowait *s_iowait) */ void hfi1_bad_pkey(struct hfi1_ibport *ibp, u32 key, u32 sl, u32 qp1, u32 qp2, u32 lid1, u32 lid2); -void hfi1_cap_mask_chg(struct rvt_dev_info *rdi, u8 port_num); +void hfi1_cap_mask_chg(struct rvt_dev_info *rdi, u32 port_num); void hfi1_sys_guid_chg(struct hfi1_ibport *ibp); void hfi1_node_desc_chg(struct hfi1_ibport *ibp); -int hfi1_process_mad(struct ib_device *ibdev, int mad_flags, u8 port, +int hfi1_process_mad(struct ib_device *ibdev, int mad_flags, u32 port, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const struct ib_mad *in_mad, struct ib_mad *out_mad, size_t *out_mad_size, u16 *out_mad_pkey_index); diff --git a/drivers/infiniband/hw/hfi1/verbs_txreq.h b/drivers/infiniband/hw/hfi1/verbs_txreq.h index d2d526c5a756..4bdfc7932376 100644 --- a/drivers/infiniband/hw/hfi1/verbs_txreq.h +++ b/drivers/infiniband/hw/hfi1/verbs_txreq.h @@ -99,11 +99,6 @@ static inline struct verbs_txreq *get_txreq(struct hfi1_ibdev *dev, return tx; } -static inline struct sdma_txreq *get_sdma_txreq(struct verbs_txreq *tx) -{ - return &tx->txreq; -} - static inline struct verbs_txreq *get_waiting_verbs_txreq(struct iowait_work *w) { struct sdma_txreq *stx; diff --git a/drivers/infiniband/hw/hfi1/vnic.h b/drivers/infiniband/hw/hfi1/vnic.h index 66150a13f374..a7a450e2cf2c 100644 --- a/drivers/infiniband/hw/hfi1/vnic.h +++ b/drivers/infiniband/hw/hfi1/vnic.h @@ -156,7 +156,7 @@ bool hfi1_vnic_sdma_write_avail(struct hfi1_vnic_vport_info *vinfo, /* vnic rdma netdev operations */ struct net_device *hfi1_vnic_alloc_rn(struct ib_device *device, - u8 port_num, + u32 port_num, enum rdma_netdev_t type, const char *name, unsigned char name_assign_type, diff --git a/drivers/infiniband/hw/hfi1/vnic_main.c b/drivers/infiniband/hw/hfi1/vnic_main.c index a90824de0f57..7e79c0578ecf 100644 --- a/drivers/infiniband/hw/hfi1/vnic_main.c +++ b/drivers/infiniband/hw/hfi1/vnic_main.c @@ -593,7 +593,7 @@ static void hfi1_vnic_free_rn(struct net_device *netdev) } struct net_device *hfi1_vnic_alloc_rn(struct ib_device *device, - u8 port_num, + u32 port_num, enum rdma_netdev_t type, const char *name, unsigned char name_assign_type, diff --git a/drivers/infiniband/hw/hns/hns_roce_alloc.c b/drivers/infiniband/hw/hns/hns_roce_alloc.c index 4bcaaa0524b1..5d389ed55376 100644 --- a/drivers/infiniband/hw/hns/hns_roce_alloc.c +++ b/drivers/infiniband/hw/hns/hns_roce_alloc.c @@ -304,6 +304,9 @@ done: void hns_roce_cleanup_bitmap(struct hns_roce_dev *hr_dev) { + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC) + hns_roce_cleanup_xrcd_table(hr_dev); + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) hns_roce_cleanup_srq_table(hr_dev); hns_roce_cleanup_qp_table(hr_dev); diff --git a/drivers/infiniband/hw/hns/hns_roce_cmd.c b/drivers/infiniband/hw/hns/hns_roce_cmd.c index 339e3fd98b0b..8f68cc3ff193 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cmd.c +++ b/drivers/infiniband/hw/hns/hns_roce_cmd.c @@ -38,22 +38,14 @@ #define CMD_POLL_TOKEN 0xffff #define CMD_MAX_NUM 32 -#define CMD_TOKEN_MASK 0x1f static int hns_roce_cmd_mbox_post_hw(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param, u32 in_modifier, u8 op_modifier, u16 op, u16 token, int event) { - struct hns_roce_cmdq *cmd = &hr_dev->cmd; - int ret; - - mutex_lock(&cmd->hcr_mutex); - ret = hr_dev->hw->post_mbox(hr_dev, in_param, out_param, in_modifier, - op_modifier, op, token, event); - mutex_unlock(&cmd->hcr_mutex); - - return ret; + return hr_dev->hw->post_mbox(hr_dev, in_param, out_param, in_modifier, + op_modifier, op, token, event); } /* this should be called with "poll_sem" */ @@ -62,18 +54,19 @@ static int __hns_roce_cmd_mbox_poll(struct hns_roce_dev *hr_dev, u64 in_param, u8 op_modifier, u16 op, unsigned int timeout) { - struct device *dev = hr_dev->dev; int ret; ret = hns_roce_cmd_mbox_post_hw(hr_dev, in_param, out_param, in_modifier, op_modifier, op, CMD_POLL_TOKEN, 0); if (ret) { - dev_err(dev, "[cmd_poll]hns_roce_cmd_mbox_post_hw failed\n"); + dev_err_ratelimited(hr_dev->dev, + "failed to post mailbox %x in poll mode, ret = %d.\n", + op, ret); return ret; } - return hr_dev->hw->chk_mbox(hr_dev, timeout); + return hr_dev->hw->poll_mbox_done(hr_dev, timeout); } static int hns_roce_cmd_mbox_poll(struct hns_roce_dev *hr_dev, u64 in_param, @@ -96,15 +89,18 @@ void hns_roce_cmd_event(struct hns_roce_dev *hr_dev, u16 token, u8 status, struct hns_roce_cmd_context *context = &hr_dev->cmd.context[token % hr_dev->cmd.max_cmds]; - if (token != context->token) + if (unlikely(token != context->token)) { + dev_err_ratelimited(hr_dev->dev, + "[cmd] invalid ae token %x,context token is %x!\n", + token, context->token); return; + } context->result = (status == HNS_ROCE_CMD_SUCCESS) ? 0 : (-EIO); context->out_param = out_param; complete(&context->done); } -/* this should be called with "use_events" */ static int __hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param, unsigned long in_modifier, u8 op_modifier, u16 op, @@ -116,44 +112,44 @@ static int __hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param, int ret; spin_lock(&cmd->context_lock); - WARN_ON(cmd->free_head < 0); - context = &cmd->context[cmd->free_head]; - context->token += cmd->token_mask + 1; - cmd->free_head = context->next; + + do { + context = &cmd->context[cmd->free_head]; + cmd->free_head = context->next; + } while (context->busy); + + context->busy = 1; + context->token += cmd->max_cmds; + spin_unlock(&cmd->context_lock); - init_completion(&context->done); + reinit_completion(&context->done); ret = hns_roce_cmd_mbox_post_hw(hr_dev, in_param, out_param, in_modifier, op_modifier, op, context->token, 1); - if (ret) + if (ret) { + dev_err_ratelimited(dev, + "failed to post mailbox %x in event mode, ret = %d.\n", + op, ret); goto out; + } - /* - * It is timeout when wait_for_completion_timeout return 0 - * The return value is the time limit set in advance - * how many seconds showing - */ if (!wait_for_completion_timeout(&context->done, msecs_to_jiffies(timeout))) { - dev_err(dev, "[cmd]wait_for_completion_timeout timeout\n"); + dev_err_ratelimited(dev, "[cmd] token %x mailbox %x timeout.\n", + context->token, op); ret = -EBUSY; goto out; } ret = context->result; - if (ret) { - dev_err(dev, "[cmd]event mod cmd process error!err=%d\n", ret); - goto out; - } + if (ret) + dev_err_ratelimited(dev, "[cmd] token %x mailbox %x error %d\n", + context->token, op, ret); out: - spin_lock(&cmd->context_lock); - context->next = cmd->free_head; - cmd->free_head = context - cmd->context; - spin_unlock(&cmd->context_lock); - + context->busy = 0; return ret; } @@ -175,44 +171,28 @@ int hns_roce_cmd_mbox(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param, unsigned long in_modifier, u8 op_modifier, u16 op, unsigned int timeout) { - int ret; + bool is_busy; - if (hr_dev->hw->rst_prc_mbox) { - ret = hr_dev->hw->rst_prc_mbox(hr_dev); - if (ret == CMD_RST_PRC_SUCCESS) - return 0; - else if (ret == CMD_RST_PRC_EBUSY) - return -EBUSY; - } + if (hr_dev->hw->chk_mbox_avail) + if (!hr_dev->hw->chk_mbox_avail(hr_dev, &is_busy)) + return is_busy ? -EBUSY : 0; if (hr_dev->cmd.use_events) - ret = hns_roce_cmd_mbox_wait(hr_dev, in_param, out_param, - in_modifier, op_modifier, op, - timeout); + return hns_roce_cmd_mbox_wait(hr_dev, in_param, out_param, + in_modifier, op_modifier, op, + timeout); else - ret = hns_roce_cmd_mbox_poll(hr_dev, in_param, out_param, - in_modifier, op_modifier, op, - timeout); - - if (ret == CMD_RST_PRC_EBUSY) - return -EBUSY; - - if (ret && (hr_dev->hw->rst_prc_mbox && - hr_dev->hw->rst_prc_mbox(hr_dev) == CMD_RST_PRC_SUCCESS)) - return 0; - - return ret; + return hns_roce_cmd_mbox_poll(hr_dev, in_param, out_param, + in_modifier, op_modifier, op, + timeout); } int hns_roce_cmd_init(struct hns_roce_dev *hr_dev) { - struct device *dev = hr_dev->dev; - - mutex_init(&hr_dev->cmd.hcr_mutex); sema_init(&hr_dev->cmd.poll_sem, 1); hr_dev->cmd.use_events = 0; hr_dev->cmd.max_cmds = CMD_MAX_NUM; - hr_dev->cmd.pool = dma_pool_create("hns_roce_cmd", dev, + hr_dev->cmd.pool = dma_pool_create("hns_roce_cmd", hr_dev->dev, HNS_ROCE_MAILBOX_SIZE, HNS_ROCE_MAILBOX_SIZE, 0); if (!hr_dev->cmd.pool) @@ -239,16 +219,16 @@ int hns_roce_cmd_use_events(struct hns_roce_dev *hr_dev) for (i = 0; i < hr_cmd->max_cmds; ++i) { hr_cmd->context[i].token = i; hr_cmd->context[i].next = i + 1; + init_completion(&hr_cmd->context[i].done); } - - hr_cmd->context[hr_cmd->max_cmds - 1].next = -1; + hr_cmd->context[hr_cmd->max_cmds - 1].next = 0; hr_cmd->free_head = 0; sema_init(&hr_cmd->event_sem, hr_cmd->max_cmds); spin_lock_init(&hr_cmd->context_lock); - hr_cmd->token_mask = CMD_TOKEN_MASK; hr_cmd->use_events = 1; + down(&hr_cmd->poll_sem); return 0; } @@ -259,6 +239,8 @@ void hns_roce_cmd_use_polling(struct hns_roce_dev *hr_dev) kfree(hr_cmd->context); hr_cmd->use_events = 0; + + up(&hr_cmd->poll_sem); } struct hns_roce_cmd_mailbox * diff --git a/drivers/infiniband/hw/hns/hns_roce_common.h b/drivers/infiniband/hw/hns/hns_roce_common.h index 23c438cef40d..d5fe56c78394 100644 --- a/drivers/infiniband/hw/hns/hns_roce_common.h +++ b/drivers/infiniband/hw/hns/hns_roce_common.h @@ -48,7 +48,8 @@ #define roce_set_field(origin, mask, shift, val) \ do { \ (origin) &= ~cpu_to_le32(mask); \ - (origin) |= cpu_to_le32(((u32)(val) << (u32)(shift)) & (mask)); \ + (origin) |= \ + cpu_to_le32(((u32)(val) << (u32)(shift)) & (mask)); \ } while (0) #define roce_set_bit(origin, shift, val) \ @@ -59,9 +60,9 @@ #define _hr_reg_enable(ptr, field_type, field_h, field_l) \ ({ \ const field_type *_ptr = ptr; \ - *((__le32 *)_ptr + (field_h) / 32) |= \ - cpu_to_le32(BIT((field_l) % 32)) + \ - BUILD_BUG_ON_ZERO((field_h) != (field_l)); \ + *((__le32 *)_ptr + (field_h) / 32) |= cpu_to_le32( \ + BIT((field_l) % 32) + \ + BUILD_BUG_ON_ZERO((field_h) != (field_l))); \ }) #define hr_reg_enable(ptr, field) _hr_reg_enable(ptr, field) @@ -69,11 +70,9 @@ #define _hr_reg_clear(ptr, field_type, field_h, field_l) \ ({ \ const field_type *_ptr = ptr; \ + BUILD_BUG_ON(((field_h) / 32) != ((field_l) / 32)); \ *((__le32 *)_ptr + (field_h) / 32) &= \ - cpu_to_le32( \ - ~GENMASK((field_h) % 32, (field_l) % 32)) + \ - BUILD_BUG_ON_ZERO(((field_h) / 32) != \ - ((field_l) / 32)); \ + ~cpu_to_le32(GENMASK((field_h) % 32, (field_l) % 32)); \ }) #define hr_reg_clear(ptr, field) _hr_reg_clear(ptr, field) @@ -87,6 +86,16 @@ #define hr_reg_write(ptr, field, val) _hr_reg_write(ptr, field, val) +#define _hr_reg_read(ptr, field_type, field_h, field_l) \ + ({ \ + const field_type *_ptr = ptr; \ + BUILD_BUG_ON(((field_h) / 32) != ((field_l) / 32)); \ + FIELD_GET(GENMASK((field_h) % 32, (field_l) % 32), \ + le32_to_cpu(*((__le32 *)_ptr + (field_h) / 32))); \ + }) + +#define hr_reg_read(ptr, field) _hr_reg_read(ptr, field) + #define ROCEE_GLB_CFG_ROCEE_DB_SQ_MODE_S 3 #define ROCEE_GLB_CFG_ROCEE_DB_OTH_MODE_S 4 diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index 74fc4940b03a..800884b074f2 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -225,7 +225,7 @@ static int alloc_cq_db(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, struct ib_udata *udata, unsigned long addr, struct hns_roce_ib_create_cq_resp *resp) { - bool has_db = hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB; + bool has_db = hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQ_RECORD_DB; struct hns_roce_ucontext *uctx; int err; @@ -250,8 +250,8 @@ static int alloc_cq_db(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, *hr_cq->set_ci_db = 0; hr_cq->flags |= HNS_ROCE_CQ_FLAG_RECORD_DB; } - hr_cq->cq_db_l = hr_dev->reg_base + hr_dev->odb_offset + - DB_REG_OFFSET * hr_dev->priv_uar.index; + hr_cq->db_reg = hr_dev->reg_base + hr_dev->odb_offset + + DB_REG_OFFSET * hr_dev->priv_uar.index; } return 0; @@ -276,6 +276,57 @@ static void free_cq_db(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, } } +static int verify_cq_create_attr(struct hns_roce_dev *hr_dev, + const struct ib_cq_init_attr *attr) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + + if (!attr->cqe || attr->cqe > hr_dev->caps.max_cqes) { + ibdev_err(ibdev, "failed to check CQ count %u, max = %u.\n", + attr->cqe, hr_dev->caps.max_cqes); + return -EINVAL; + } + + if (attr->comp_vector >= hr_dev->caps.num_comp_vectors) { + ibdev_err(ibdev, "failed to check CQ vector = %u, max = %d.\n", + attr->comp_vector, hr_dev->caps.num_comp_vectors); + return -EINVAL; + } + + return 0; +} + +static int get_cq_ucmd(struct hns_roce_cq *hr_cq, struct ib_udata *udata, + struct hns_roce_ib_create_cq *ucmd) +{ + struct ib_device *ibdev = hr_cq->ib_cq.device; + int ret; + + ret = ib_copy_from_udata(ucmd, udata, min(udata->inlen, sizeof(*ucmd))); + if (ret) { + ibdev_err(ibdev, "failed to copy CQ udata, ret = %d.\n", ret); + return ret; + } + + return 0; +} + +static void set_cq_param(struct hns_roce_cq *hr_cq, u32 cq_entries, int vector, + struct hns_roce_ib_create_cq *ucmd) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device); + + cq_entries = max(cq_entries, hr_dev->caps.min_cqes); + cq_entries = roundup_pow_of_two(cq_entries); + hr_cq->ib_cq.cqe = cq_entries - 1; /* used as cqe index */ + hr_cq->cq_depth = cq_entries; + hr_cq->vector = vector; + + spin_lock_init(&hr_cq->lock); + INIT_LIST_HEAD(&hr_cq->sq_list); + INIT_LIST_HEAD(&hr_cq->rq_list); +} + static void set_cqe_size(struct hns_roce_cq *hr_cq, struct ib_udata *udata, struct hns_roce_ib_create_cq *ucmd) { @@ -299,44 +350,23 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr, struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq); struct ib_device *ibdev = &hr_dev->ib_dev; struct hns_roce_ib_create_cq ucmd = {}; - int vector = attr->comp_vector; - u32 cq_entries = attr->cqe; int ret; if (attr->flags) return -EOPNOTSUPP; - if (cq_entries < 1 || cq_entries > hr_dev->caps.max_cqes) { - ibdev_err(ibdev, "failed to check CQ count %u, max = %u.\n", - cq_entries, hr_dev->caps.max_cqes); - return -EINVAL; - } - - if (vector >= hr_dev->caps.num_comp_vectors) { - ibdev_err(ibdev, "failed to check CQ vector = %d, max = %d.\n", - vector, hr_dev->caps.num_comp_vectors); - return -EINVAL; - } - - cq_entries = max(cq_entries, hr_dev->caps.min_cqes); - cq_entries = roundup_pow_of_two(cq_entries); - hr_cq->ib_cq.cqe = cq_entries - 1; /* used as cqe index */ - hr_cq->cq_depth = cq_entries; - hr_cq->vector = vector; - spin_lock_init(&hr_cq->lock); - INIT_LIST_HEAD(&hr_cq->sq_list); - INIT_LIST_HEAD(&hr_cq->rq_list); + ret = verify_cq_create_attr(hr_dev, attr); + if (ret) + return ret; if (udata) { - ret = ib_copy_from_udata(&ucmd, udata, - min(udata->inlen, sizeof(ucmd))); - if (ret) { - ibdev_err(ibdev, "failed to copy CQ udata, ret = %d.\n", - ret); + ret = get_cq_ucmd(hr_cq, udata, &ucmd); + if (ret) return ret; - } } + set_cq_param(hr_cq, attr->cqe, attr->comp_vector, &ucmd); + set_cqe_size(hr_cq, udata, &ucmd); ret = alloc_cq_buf(hr_dev, hr_cq, udata, ucmd.buf_addr); diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 3d6b7a2db496..97800d2b9d39 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -137,6 +137,7 @@ enum { SERV_TYPE_UC, SERV_TYPE_RD, SERV_TYPE_UD, + SERV_TYPE_XRC = 5, }; enum hns_roce_qp_state { @@ -168,6 +169,8 @@ enum hns_roce_event { HNS_ROCE_EVENT_TYPE_DB_OVERFLOW = 0x12, HNS_ROCE_EVENT_TYPE_MB = 0x13, HNS_ROCE_EVENT_TYPE_FLR = 0x15, + HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION = 0x16, + HNS_ROCE_EVENT_TYPE_INVALID_XRCETH = 0x17, }; #define HNS_ROCE_CAP_FLAGS_EX_SHIFT 12 @@ -176,9 +179,10 @@ enum { HNS_ROCE_CAP_FLAG_REREG_MR = BIT(0), HNS_ROCE_CAP_FLAG_ROCE_V1_V2 = BIT(1), HNS_ROCE_CAP_FLAG_RQ_INLINE = BIT(2), - HNS_ROCE_CAP_FLAG_RECORD_DB = BIT(3), - HNS_ROCE_CAP_FLAG_SQ_RECORD_DB = BIT(4), + HNS_ROCE_CAP_FLAG_CQ_RECORD_DB = BIT(3), + HNS_ROCE_CAP_FLAG_QP_RECORD_DB = BIT(4), HNS_ROCE_CAP_FLAG_SRQ = BIT(5), + HNS_ROCE_CAP_FLAG_XRC = BIT(6), HNS_ROCE_CAP_FLAG_MW = BIT(7), HNS_ROCE_CAP_FLAG_FRMR = BIT(8), HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL = BIT(9), @@ -214,12 +218,6 @@ enum { HNS_ROCE_RST_DIRECT_RETURN = 0, }; -enum { - CMD_RST_PRC_OTHERS, - CMD_RST_PRC_SUCCESS, - CMD_RST_PRC_EBUSY, -}; - #define HNS_ROCE_CMD_SUCCESS 1 /* The minimum page size is 4K for hardware */ @@ -244,6 +242,11 @@ struct hns_roce_pd { unsigned long pdn; }; +struct hns_roce_xrcd { + struct ib_xrcd ibxrcd; + u32 xrcdn; +}; + struct hns_roce_bitmap { /* Bitmap Traversal last a bit which is 1 */ unsigned long last; @@ -363,7 +366,7 @@ struct hns_roce_wq { int wqe_shift; /* WQE size */ u32 head; u32 tail; - void __iomem *db_reg_l; + void __iomem *db_reg; }; struct hns_roce_sge { @@ -437,7 +440,7 @@ struct hns_roce_cq { u32 cq_depth; u32 cons_index; u32 *set_ci_db; - void __iomem *cq_db_l; + void __iomem *db_reg; u16 *tptr_addr; int arm_sn; int cqe_size; @@ -467,7 +470,8 @@ struct hns_roce_srq { u32 rsv_sge; int wqe_shift; u32 cqn; - void __iomem *db_reg_l; + u32 xrcdn; + void __iomem *db_reg; atomic_t refcount; struct completion free; @@ -546,6 +550,7 @@ struct hns_roce_cmd_context { int next; u64 out_param; u16 token; + u16 busy; }; struct hns_roce_cmdq { @@ -562,11 +567,6 @@ struct hns_roce_cmdq { int free_head; struct hns_roce_cmd_context *context; /* - * Result of get integer part - * which max_comds compute according a power of 2 - */ - u16 token_mask; - /* * Process whether use event mode, init default non-zero * After the event queue of cmd event ready, * can switch into event mode @@ -640,6 +640,8 @@ struct hns_roce_qp { enum hns_roce_event event_type); unsigned long qpn; + u32 xrcdn; + atomic_t refcount; struct completion free; @@ -695,7 +697,7 @@ struct hns_roce_aeqe { struct hns_roce_eq { struct hns_roce_dev *hr_dev; - void __iomem *doorbell; + void __iomem *db_reg; int type_flag; /* Aeq:1 ceq:0 */ int eqn; @@ -723,6 +725,13 @@ struct hns_roce_eq_table { void __iomem **eqc_base; /* only for hw v1 */ }; +enum cong_type { + CONG_TYPE_DCQCN, + CONG_TYPE_LDCP, + CONG_TYPE_HC3, + CONG_TYPE_DIP, +}; + struct hns_roce_caps { u64 fw_ver; u8 num_ports; @@ -759,13 +768,14 @@ struct hns_roce_caps { int num_other_vectors; u32 num_mtpts; u32 num_mtt_segs; - u32 num_cqe_segs; u32 num_srqwqe_segs; u32 num_idx_segs; int reserved_mrws; int reserved_uars; int num_pds; int reserved_pds; + u32 num_xrcds; + u32 reserved_xrcds; u32 mtt_entry_sz; u32 cqe_sz; u32 page_size_cap; @@ -794,6 +804,9 @@ struct hns_roce_caps { u32 cqc_bt_num; u32 cqc_timer_bt_num; u32 mpt_bt_num; + u32 eqc_bt_num; + u32 smac_bt_num; + u32 sgid_bt_num; u32 sccc_bt_num; u32 gmv_bt_num; u32 qpc_ba_pg_sz; @@ -851,6 +864,7 @@ struct hns_roce_caps { u16 default_aeq_period; u16 default_aeq_arm_st; u16 default_ceq_arm_st; + enum cong_type cong_type; }; struct hns_roce_dfx_hw { @@ -874,9 +888,10 @@ struct hns_roce_hw { int (*post_mbox)(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param, u32 in_modifier, u8 op_modifier, u16 op, u16 token, int event); - int (*chk_mbox)(struct hns_roce_dev *hr_dev, unsigned int timeout); - int (*rst_prc_mbox)(struct hns_roce_dev *hr_dev); - int (*set_gid)(struct hns_roce_dev *hr_dev, u8 port, int gid_index, + int (*poll_mbox_done)(struct hns_roce_dev *hr_dev, + unsigned int timeout); + bool (*chk_mbox_avail)(struct hns_roce_dev *hr_dev, bool *is_busy); + int (*set_gid)(struct hns_roce_dev *hr_dev, u32 port, int gid_index, const union ib_gid *gid, const struct ib_gid_attr *attr); int (*set_mac)(struct hns_roce_dev *hr_dev, u8 phy_port, u8 *addr); void (*set_mtu)(struct hns_roce_dev *hr_dev, u8 phy_port, @@ -897,33 +912,17 @@ struct hns_roce_hw { int (*clear_hem)(struct hns_roce_dev *hr_dev, struct hns_roce_hem_table *table, int obj, int step_idx); - int (*query_qp)(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, - int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr); int (*modify_qp)(struct ib_qp *ibqp, const struct ib_qp_attr *attr, int attr_mask, enum ib_qp_state cur_state, enum ib_qp_state new_state); - int (*destroy_qp)(struct ib_qp *ibqp, struct ib_udata *udata); int (*qp_flow_control_init)(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp); - int (*post_send)(struct ib_qp *ibqp, const struct ib_send_wr *wr, - const struct ib_send_wr **bad_wr); - int (*post_recv)(struct ib_qp *qp, const struct ib_recv_wr *recv_wr, - const struct ib_recv_wr **bad_recv_wr); - int (*req_notify_cq)(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); - int (*poll_cq)(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); int (*dereg_mr)(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr, struct ib_udata *udata); int (*destroy_cq)(struct ib_cq *ibcq, struct ib_udata *udata); - int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period); int (*init_eq)(struct hns_roce_dev *hr_dev); void (*cleanup_eq)(struct hns_roce_dev *hr_dev); int (*write_srqc)(struct hns_roce_srq *srq, void *mb_buf); - int (*modify_srq)(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr, - enum ib_srq_attr_mask srq_attr_mask, - struct ib_udata *udata); - int (*query_srq)(struct ib_srq *ibsrq, struct ib_srq_attr *attr); - int (*post_srq_recv)(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, - const struct ib_recv_wr **bad_wr); const struct ib_device_ops *hns_roce_dev_ops; const struct ib_device_ops *hns_roce_dev_srq_ops; }; @@ -945,6 +944,8 @@ struct hns_roce_dev { enum hns_roce_device_state state; struct list_head qp_list; /* list of all qps on this dev */ spinlock_t qp_list_lock; /* protect qp_list */ + struct list_head dip_list; /* list of all dest ips on this dev */ + spinlock_t dip_list_lock; /* protect dip_list */ struct list_head pgdir_list; struct mutex pgdir_mutex; @@ -963,6 +964,7 @@ struct hns_roce_dev { struct hns_roce_cmdq cmd; struct hns_roce_bitmap pd_bitmap; + struct hns_roce_bitmap xrcd_bitmap; struct hns_roce_uar_table uar_table; struct hns_roce_mr_table mr_table; struct hns_roce_cq_table cq_table; @@ -986,6 +988,9 @@ struct hns_roce_dev { void *priv; struct workqueue_struct *irq_workq; const struct hns_roce_dfx_hw *dfx; + u32 func_num; + u32 is_vf; + u32 cong_algo_tmpl_id; }; static inline struct hns_roce_dev *to_hr_dev(struct ib_device *ib_dev) @@ -1004,6 +1009,11 @@ static inline struct hns_roce_pd *to_hr_pd(struct ib_pd *ibpd) return container_of(ibpd, struct hns_roce_pd, ibpd); } +static inline struct hns_roce_xrcd *to_hr_xrcd(struct ib_xrcd *ibxrcd) +{ + return container_of(ibxrcd, struct hns_roce_xrcd, ibxrcd); +} + static inline struct hns_roce_ah *to_hr_ah(struct ib_ah *ibah) { return container_of(ibah, struct hns_roce_ah, ibah); @@ -1136,6 +1146,7 @@ int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev); void hns_roce_init_cq_table(struct hns_roce_dev *hr_dev); int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev); int hns_roce_init_srq_table(struct hns_roce_dev *hr_dev); +int hns_roce_init_xrcd_table(struct hns_roce_dev *hr_dev); void hns_roce_cleanup_pd_table(struct hns_roce_dev *hr_dev); void hns_roce_cleanup_mr_table(struct hns_roce_dev *hr_dev); @@ -1143,6 +1154,7 @@ void hns_roce_cleanup_eq_table(struct hns_roce_dev *hr_dev); void hns_roce_cleanup_cq_table(struct hns_roce_dev *hr_dev); void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev); void hns_roce_cleanup_srq_table(struct hns_roce_dev *hr_dev); +void hns_roce_cleanup_xrcd_table(struct hns_roce_dev *hr_dev); int hns_roce_bitmap_alloc(struct hns_roce_bitmap *bitmap, unsigned long *obj); void hns_roce_bitmap_free(struct hns_roce_bitmap *bitmap, unsigned long obj, @@ -1207,6 +1219,9 @@ int hns_roce_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr, struct ib_udata *udata); int hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata); +int hns_roce_alloc_xrcd(struct ib_xrcd *ib_xrcd, struct ib_udata *udata); +int hns_roce_dealloc_xrcd(struct ib_xrcd *ib_xrcd, struct ib_udata *udata); + struct ib_qp *hns_roce_create_qp(struct ib_pd *ib_pd, struct ib_qp_init_attr *init_attr, struct ib_udata *udata); @@ -1246,7 +1261,7 @@ void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn); void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type); void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type); void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type); -u8 hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index); +u8 hns_get_gid_index(struct hns_roce_dev *hr_dev, u32 port, int gid_index); void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev); int hns_roce_init(struct hns_roce_dev *hr_dev); void hns_roce_exit(struct hns_roce_dev *hr_dev); diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c index 5346fdca9473..620acf66b22c 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c @@ -54,7 +54,7 @@ * GID[0][0], GID[1][0],.....GID[N - 1][0], * And so on */ -u8 hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index) +u8 hns_get_gid_index(struct hns_roce_dev *hr_dev, u32 port, int gid_index) { return gid_index * hr_dev->caps.num_ports + port; } @@ -345,7 +345,7 @@ out: doorbell[0] = sq_db.u32_4; doorbell[1] = sq_db.u32_8; - hns_roce_write64_k(doorbell, qp->sq.db_reg_l); + hns_roce_write64_k(doorbell, qp->sq.db_reg); } spin_unlock_irqrestore(&qp->sq.lock, flags); @@ -440,7 +440,7 @@ out: doorbell[0] = rq_db.u32_4; doorbell[1] = rq_db.u32_8; - hns_roce_write64_k(doorbell, hr_qp->rq.db_reg_l); + hns_roce_write64_k(doorbell, hr_qp->rq.db_reg); } } spin_unlock_irqrestore(&hr_qp->rq.lock, flags); @@ -538,7 +538,7 @@ static void hns_roce_set_sdb_ext(struct hns_roce_dev *hr_dev, u32 ext_sdb_alept, /* * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of * using 4K page, and shift more 32 because of - * caculating the high 32 bit value evaluated to hardware. + * calculating the high 32 bit value evaluated to hardware. */ roce_set_field(tmp, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_M, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S, sdb_dma_addr >> 44); @@ -711,7 +711,7 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev) int i, j; u8 queue_en[HNS_ROCE_V1_RESV_QP] = { 0 }; u8 phy_port; - u8 port = 0; + u32 port = 0; u8 sl; /* Reserved cq for loop qp */ @@ -1189,7 +1189,7 @@ static int hns_roce_raq_init(struct hns_roce_dev *hr_dev) /* * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of * using 4K page, and shift more 32 because of - * caculating the high 32 bit value evaluated to hardware. + * calculating the high 32 bit value evaluated to hardware. */ roce_set_field(tmp, ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_M, ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S, @@ -1382,7 +1382,6 @@ static int hns_roce_free_mr_init(struct hns_roce_dev *hr_dev) ret = hns_roce_v1_rsv_lp_qp(hr_dev); if (ret) { dev_err(dev, "Reserved loop qp failed(%d)!\n", ret); - flush_workqueue(free_mr->free_mr_wq); destroy_workqueue(free_mr->free_mr_wq); } @@ -1394,7 +1393,6 @@ static void hns_roce_free_mr_free(struct hns_roce_dev *hr_dev) struct hns_roce_v1_priv *priv = hr_dev->priv; struct hns_roce_free_mr *free_mr = &priv->free_mr; - flush_workqueue(free_mr->free_mr_wq); destroy_workqueue(free_mr->free_mr_wq); hns_roce_v1_release_lp_qp(hr_dev); @@ -1676,7 +1674,7 @@ static int hns_roce_v1_chk_mbox(struct hns_roce_dev *hr_dev, return 0; } -static int hns_roce_v1_set_gid(struct hns_roce_dev *hr_dev, u8 port, +static int hns_roce_v1_set_gid(struct hns_roce_dev *hr_dev, u32 port, int gid_index, const union ib_gid *gid, const struct ib_gid_attr *attr) { @@ -1939,7 +1937,7 @@ static void hns_roce_v1_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index) roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M, ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S, hr_cq->cqn); - hns_roce_write64_k(doorbell, hr_cq->cq_db_l); + hns_roce_write64_k(doorbell, hr_cq->db_reg); } static void __hns_roce_v1_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, @@ -2041,7 +2039,7 @@ static void hns_roce_v1_write_cqc(struct hns_roce_dev *hr_dev, /** * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of * using 4K page, and shift more 32 because of - * caculating the high 32 bit value evaluated to hardware. + * calculating the high 32 bit value evaluated to hardware. */ roce_set_field(cq_context->cqc_byte_20, CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_M, @@ -2092,7 +2090,7 @@ static int hns_roce_v1_req_notify_cq(struct ib_cq *ibcq, ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S, hr_cq->cqn | notification_flag); - hns_roce_write64_k(doorbell, hr_cq->cq_db_l); + hns_roce_write64_k(doorbell, hr_cq->db_reg); return 0; } @@ -2673,8 +2671,8 @@ static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, int ret = -EINVAL; u64 sq_ba = 0; u64 rq_ba = 0; - int port; - u8 port_num; + u32 port; + u32 port_num; u8 *dmac; u8 *smac; @@ -3217,12 +3215,12 @@ static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, roce_set_bit(doorbell[1], RQ_DOORBELL_U32_8_HW_SYNC_S, 1); if (ibqp->uobject) { - hr_qp->rq.db_reg_l = hr_dev->reg_base + + hr_qp->rq.db_reg = hr_dev->reg_base + hr_dev->odb_offset + DB_REG_OFFSET * hr_dev->priv_uar.index; } - hns_roce_write64_k(doorbell, hr_qp->rq.db_reg_l); + hns_roce_write64_k(doorbell, hr_qp->rq.db_reg); } hr_qp->state = new_state; @@ -3449,8 +3447,7 @@ static int hns_roce_v1_q_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, ((roce_get_bit(context->qpc_bytes_4, QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S)) << 3); - if (hr_qp->ibqp.qp_type == IB_QPT_RC || - hr_qp->ibqp.qp_type == IB_QPT_UC) { + if (hr_qp->ibqp.qp_type == IB_QPT_RC) { struct ib_global_route *grh = rdma_ah_retrieve_grh(&qp_attr->ah_attr); @@ -3604,7 +3601,7 @@ static int hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) static void set_eq_cons_index_v1(struct hns_roce_eq *eq, u32 req_not) { roce_raw_write((eq->cons_index & HNS_ROCE_V1_CONS_IDX_M) | - (req_not << eq->log_entries), eq->doorbell); + (req_not << eq->log_entries), eq->db_reg); } static void hns_roce_v1_wq_catas_err_handle(struct hns_roce_dev *hr_dev, @@ -4170,7 +4167,7 @@ static int hns_roce_v1_create_eq(struct hns_roce_dev *hr_dev, * Configure eq extended address 45~49 bit. * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of * using 4K page, and shift more 32 because of - * caculating the high 32 bit value evaluated to hardware. + * calculating the high 32 bit value evaluated to hardware. */ roce_set_field(tmp1, ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_M, ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_S, @@ -4234,9 +4231,9 @@ static int hns_roce_v1_init_eq_table(struct hns_roce_dev *hr_dev) ROCEE_CAEP_CEQC_SHIFT_0_REG + CEQ_REG_OFFSET * i; eq->type_flag = HNS_ROCE_CEQ; - eq->doorbell = hr_dev->reg_base + - ROCEE_CAEP_CEQC_CONS_IDX_0_REG + - CEQ_REG_OFFSET * i; + eq->db_reg = hr_dev->reg_base + + ROCEE_CAEP_CEQC_CONS_IDX_0_REG + + CEQ_REG_OFFSET * i; eq->entries = hr_dev->caps.ceqe_depth; eq->log_entries = ilog2(eq->entries); eq->eqe_size = HNS_ROCE_CEQE_SIZE; @@ -4245,8 +4242,8 @@ static int hns_roce_v1_init_eq_table(struct hns_roce_dev *hr_dev) eq_table->eqc_base[i] = hr_dev->reg_base + ROCEE_CAEP_AEQC_AEQE_SHIFT_REG; eq->type_flag = HNS_ROCE_AEQ; - eq->doorbell = hr_dev->reg_base + - ROCEE_CAEP_AEQE_CONS_IDX_REG; + eq->db_reg = hr_dev->reg_base + + ROCEE_CAEP_AEQE_CONS_IDX_REG; eq->entries = hr_dev->caps.aeqe_depth; eq->log_entries = ilog2(eq->entries); eq->eqe_size = HNS_ROCE_AEQE_SIZE; @@ -4349,7 +4346,7 @@ static const struct hns_roce_hw hns_roce_hw_v1 = { .hw_init = hns_roce_v1_init, .hw_exit = hns_roce_v1_exit, .post_mbox = hns_roce_v1_post_mbox, - .chk_mbox = hns_roce_v1_chk_mbox, + .poll_mbox_done = hns_roce_v1_chk_mbox, .set_gid = hns_roce_v1_set_gid, .set_mac = hns_roce_v1_set_mac, .set_mtu = hns_roce_v1_set_mtu, @@ -4357,12 +4354,6 @@ static const struct hns_roce_hw hns_roce_hw_v1 = { .write_cqc = hns_roce_v1_write_cqc, .clear_hem = hns_roce_v1_clear_hem, .modify_qp = hns_roce_v1_modify_qp, - .query_qp = hns_roce_v1_query_qp, - .destroy_qp = hns_roce_v1_destroy_qp, - .post_send = hns_roce_v1_post_send, - .post_recv = hns_roce_v1_post_recv, - .req_notify_cq = hns_roce_v1_req_notify_cq, - .poll_cq = hns_roce_v1_poll_cq, .dereg_mr = hns_roce_v1_dereg_mr, .destroy_cq = hns_roce_v1_destroy_cq, .init_eq = hns_roce_v1_init_eq_table, diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index ce26f97b2ca2..7652dafe32ec 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -48,6 +48,12 @@ #include "hns_roce_hem.h" #include "hns_roce_hw_v2.h" +enum { + CMD_RST_PRC_OTHERS, + CMD_RST_PRC_SUCCESS, + CMD_RST_PRC_EBUSY, +}; + static inline void set_data_seg_v2(struct hns_roce_v2_wqe_data_seg *dseg, struct ib_sge *sg) { @@ -632,24 +638,60 @@ static inline void update_sq_db(struct hns_roce_dev *hr_dev, * around the mailbox calls. Hence, use the deferred flush for * now. */ - if (qp->state == IB_QPS_ERR) { + if (unlikely(qp->state == IB_QPS_ERR)) { if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &qp->flush_flag)) init_flush_work(hr_dev, qp); } else { struct hns_roce_v2_db sq_db = {}; - roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_TAG_M, - V2_DB_BYTE_4_TAG_S, qp->doorbell_qpn); - roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_CMD_M, - V2_DB_BYTE_4_CMD_S, HNS_ROCE_V2_SQ_DB); + roce_set_field(sq_db.byte_4, V2_DB_TAG_M, V2_DB_TAG_S, + qp->doorbell_qpn); + roce_set_field(sq_db.byte_4, V2_DB_CMD_M, V2_DB_CMD_S, + HNS_ROCE_V2_SQ_DB); + /* indicates data on new BAR, 0 : SQ doorbell, 1 : DWQE */ roce_set_bit(sq_db.byte_4, V2_DB_FLAG_S, 0); - roce_set_field(sq_db.parameter, V2_DB_PARAMETER_IDX_M, - V2_DB_PARAMETER_IDX_S, qp->sq.head); - roce_set_field(sq_db.parameter, V2_DB_PARAMETER_SL_M, - V2_DB_PARAMETER_SL_S, qp->sl); + roce_set_field(sq_db.parameter, V2_DB_PRODUCER_IDX_M, + V2_DB_PRODUCER_IDX_S, qp->sq.head); + roce_set_field(sq_db.parameter, V2_DB_SL_M, V2_DB_SL_S, + qp->sl); + + hns_roce_write64(hr_dev, (__le32 *)&sq_db, qp->sq.db_reg); + } +} + +static inline void update_rq_db(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *qp) +{ + /* + * Hip08 hardware cannot flush the WQEs in RQ if the QP state + * gets into errored mode. Hence, as a workaround to this + * hardware limitation, driver needs to assist in flushing. But + * the flushing operation uses mailbox to convey the QP state to + * the hardware and which can sleep due to the mutex protection + * around the mailbox calls. Hence, use the deferred flush for + * now. + */ + if (unlikely(qp->state == IB_QPS_ERR)) { + if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &qp->flush_flag)) + init_flush_work(hr_dev, qp); + } else { + if (likely(qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)) { + *qp->rdb.db_record = + qp->rq.head & V2_DB_PRODUCER_IDX_M; + } else { + struct hns_roce_v2_db rq_db = {}; - hns_roce_write64(hr_dev, (__le32 *)&sq_db, qp->sq.db_reg_l); + roce_set_field(rq_db.byte_4, V2_DB_TAG_M, V2_DB_TAG_S, + qp->qpn); + roce_set_field(rq_db.byte_4, V2_DB_CMD_M, V2_DB_CMD_S, + HNS_ROCE_V2_RQ_DB); + roce_set_field(rq_db.parameter, V2_DB_PRODUCER_IDX_M, + V2_DB_PRODUCER_IDX_S, qp->rq.head); + + hns_roce_write64(hr_dev, (__le32 *)&rq_db, + qp->rq.db_reg); + } } } @@ -681,8 +723,7 @@ static void write_dwqe(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp, roce_set_field(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_WQE_INDEX_M, V2_RC_SEND_WQE_BYTE_4_WQE_INDEX_S, qp->sq.head); - hns_roce_write512(hr_dev, wqe, hr_dev->mem_base + - HNS_ROCE_DWQE_SIZE * qp->ibqp.qp_num); + hns_roce_write512(hr_dev, wqe, qp->sq.db_reg); } static int hns_roce_v2_post_send(struct ib_qp *ibqp, @@ -879,22 +920,7 @@ out: if (likely(nreq)) { hr_qp->rq.head += nreq; - /* - * Hip08 hardware cannot flush the WQEs in RQ if the QP state - * gets into errored mode. Hence, as a workaround to this - * hardware limitation, driver needs to assist in flushing. But - * the flushing operation uses mailbox to convey the QP state to - * the hardware and which can sleep due to the mutex protection - * around the mailbox calls. Hence, use the deferred flush for - * now. - */ - if (hr_qp->state == IB_QPS_ERR) { - if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, - &hr_qp->flush_flag)) - init_flush_work(hr_dev, hr_qp); - } else { - *hr_qp->rdb.db_record = hr_qp->rq.head & 0xffff; - } + update_rq_db(hr_dev, hr_qp); } spin_unlock_irqrestore(&hr_qp->rq.lock, flags); @@ -1016,13 +1042,14 @@ static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq, } if (likely(nreq)) { - srq_db.byte_4 = - cpu_to_le32(HNS_ROCE_V2_SRQ_DB << V2_DB_BYTE_4_CMD_S | - (srq->srqn & V2_DB_BYTE_4_TAG_M)); - srq_db.parameter = - cpu_to_le32(srq->idx_que.head & V2_DB_PARAMETER_IDX_M); + roce_set_field(srq_db.byte_4, V2_DB_TAG_M, V2_DB_TAG_S, + srq->srqn); + roce_set_field(srq_db.byte_4, V2_DB_CMD_M, V2_DB_CMD_S, + HNS_ROCE_V2_SRQ_DB); + roce_set_field(srq_db.parameter, V2_DB_PRODUCER_IDX_M, + V2_DB_PRODUCER_IDX_S, srq->idx_que.head); - hns_roce_write64(hr_dev, (__le32 *)&srq_db, srq->db_reg_l); + hns_roce_write64(hr_dev, (__le32 *)&srq_db, srq->db_reg); } spin_unlock_irqrestore(&srq->lock, flags); @@ -1030,7 +1057,7 @@ static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq, return ret; } -static int hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev *hr_dev, +static u32 hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev *hr_dev, unsigned long instance_stage, unsigned long reset_stage) { @@ -1053,7 +1080,7 @@ static int hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev *hr_dev, return CMD_RST_PRC_SUCCESS; } -static int hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev, +static u32 hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev, unsigned long instance_stage, unsigned long reset_stage) { @@ -1081,7 +1108,7 @@ static int hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev, return CMD_RST_PRC_SUCCESS; } -static int hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev) +static u32 hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev) { struct hns_roce_v2_priv *priv = hr_dev->priv; struct hnae3_handle *handle = priv->handle; @@ -1098,10 +1125,9 @@ static int hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev) return CMD_RST_PRC_EBUSY; } -static int hns_roce_v2_rst_process_cmd(struct hns_roce_dev *hr_dev) +static u32 check_aedev_reset_status(struct hns_roce_dev *hr_dev, + struct hnae3_handle *handle) { - struct hns_roce_v2_priv *priv = hr_dev->priv; - struct hnae3_handle *handle = priv->handle; const struct hnae3_ae_ops *ops = handle->ae_algo->ops; unsigned long instance_stage; /* the current instance stage */ unsigned long reset_stage; /* the current reset stage */ @@ -1109,9 +1135,6 @@ static int hns_roce_v2_rst_process_cmd(struct hns_roce_dev *hr_dev) bool sw_resetting; bool hw_resetting; - if (hr_dev->is_reset) - return CMD_RST_PRC_SUCCESS; - /* Get information about reset from NIC driver or RoCE driver itself, * the meaning of the following variables from NIC driver are described * as below: @@ -1122,19 +1145,53 @@ static int hns_roce_v2_rst_process_cmd(struct hns_roce_dev *hr_dev) instance_stage = handle->rinfo.instance_state; reset_stage = handle->rinfo.reset_state; reset_cnt = ops->ae_dev_reset_cnt(handle); - hw_resetting = ops->get_cmdq_stat(handle); - sw_resetting = ops->ae_dev_resetting(handle); - if (reset_cnt != hr_dev->reset_cnt) return hns_roce_v2_cmd_hw_reseted(hr_dev, instance_stage, reset_stage); - else if (hw_resetting) + + hw_resetting = ops->get_cmdq_stat(handle); + if (hw_resetting) return hns_roce_v2_cmd_hw_resetting(hr_dev, instance_stage, reset_stage); - else if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT) + + sw_resetting = ops->ae_dev_resetting(handle); + if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT) return hns_roce_v2_cmd_sw_resetting(hr_dev); - return 0; + return CMD_RST_PRC_OTHERS; +} + +static bool check_device_is_in_reset(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_v2_priv *priv = hr_dev->priv; + struct hnae3_handle *handle = priv->handle; + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + + if (hr_dev->reset_cnt != ops->ae_dev_reset_cnt(handle)) + return true; + + if (ops->get_hw_reset_stat(handle)) + return true; + + if (ops->ae_dev_resetting(handle)) + return true; + + return false; +} + +static bool v2_chk_mbox_is_avail(struct hns_roce_dev *hr_dev, bool *busy) +{ + struct hns_roce_v2_priv *priv = hr_dev->priv; + u32 status; + + if (hr_dev->is_reset) + status = CMD_RST_PRC_SUCCESS; + else + status = check_aedev_reset_status(hr_dev, priv->handle); + + *busy = (status == CMD_RST_PRC_EBUSY); + + return status == CMD_RST_PRC_OTHERS; } static int hns_roce_alloc_cmq_desc(struct hns_roce_dev *hr_dev, @@ -1152,6 +1209,9 @@ static int hns_roce_alloc_cmq_desc(struct hns_roce_dev *hr_dev, ring->desc_dma_addr = 0; kfree(ring->desc); ring->desc = NULL; + + dev_err_ratelimited(hr_dev->dev, + "failed to map cmq desc addr.\n"); return -ENOMEM; } @@ -1228,14 +1288,16 @@ static int hns_roce_v2_cmq_init(struct hns_roce_dev *hr_dev) /* Init CSQ */ ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CSQ); if (ret) { - dev_err(hr_dev->dev, "Init CSQ error, ret = %d.\n", ret); + dev_err_ratelimited(hr_dev->dev, + "failed to init CSQ, ret = %d.\n", ret); return ret; } /* Init CRQ */ ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CRQ); if (ret) { - dev_err(hr_dev->dev, "Init CRQ error, ret = %d.\n", ret); + dev_err_ratelimited(hr_dev->dev, + "failed to init CRQ, ret = %d.\n", ret); goto err_crq; } @@ -1352,27 +1414,36 @@ static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev, static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev, struct hns_roce_cmq_desc *desc, int num) { - int retval; + bool busy; int ret; - ret = hns_roce_v2_rst_process_cmd(hr_dev); - if (ret == CMD_RST_PRC_SUCCESS) - return 0; - if (ret == CMD_RST_PRC_EBUSY) - return -EBUSY; + if (!v2_chk_mbox_is_avail(hr_dev, &busy)) + return busy ? -EBUSY : 0; ret = __hns_roce_cmq_send(hr_dev, desc, num); if (ret) { - retval = hns_roce_v2_rst_process_cmd(hr_dev); - if (retval == CMD_RST_PRC_SUCCESS) - return 0; - else if (retval == CMD_RST_PRC_EBUSY) - return -EBUSY; + if (!v2_chk_mbox_is_avail(hr_dev, &busy)) + return busy ? -EBUSY : 0; } return ret; } +static int config_hem_ba_to_hw(struct hns_roce_dev *hr_dev, unsigned long obj, + dma_addr_t base_addr, u16 op) +{ + struct hns_roce_cmd_mailbox *mbox = hns_roce_alloc_cmd_mailbox(hr_dev); + int ret; + + if (IS_ERR(mbox)) + return PTR_ERR(mbox); + + ret = hns_roce_cmd_mbox(hr_dev, base_addr, mbox->dma, obj, 0, op, + HNS_ROCE_CMD_TIMEOUT_MSECS); + hns_roce_free_cmd_mailbox(hr_dev, mbox); + return ret; +} + static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev) { struct hns_roce_query_version *resp; @@ -1391,92 +1462,90 @@ static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev) return 0; } -static bool hns_roce_func_clr_chk_rst(struct hns_roce_dev *hr_dev) +static void func_clr_hw_resetting_state(struct hns_roce_dev *hr_dev, + struct hnae3_handle *handle) { - struct hns_roce_v2_priv *priv = hr_dev->priv; - struct hnae3_handle *handle = priv->handle; const struct hnae3_ae_ops *ops = handle->ae_algo->ops; - unsigned long reset_cnt; - bool sw_resetting; - bool hw_resetting; + unsigned long end; - reset_cnt = ops->ae_dev_reset_cnt(handle); - hw_resetting = ops->get_hw_reset_stat(handle); - sw_resetting = ops->ae_dev_resetting(handle); + hr_dev->dis_db = true; - if (reset_cnt != hr_dev->reset_cnt || hw_resetting || sw_resetting) - return true; + dev_warn(hr_dev->dev, + "Func clear is pending, device in resetting state.\n"); + end = HNS_ROCE_V2_HW_RST_TIMEOUT; + while (end) { + if (!ops->get_hw_reset_stat(handle)) { + hr_dev->is_reset = true; + dev_info(hr_dev->dev, + "Func clear success after reset.\n"); + return; + } + msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT); + end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT; + } - return false; + dev_warn(hr_dev->dev, "Func clear failed.\n"); } -static void hns_roce_func_clr_rst_prc(struct hns_roce_dev *hr_dev, int retval, - int flag) +static void func_clr_sw_resetting_state(struct hns_roce_dev *hr_dev, + struct hnae3_handle *handle) { - struct hns_roce_v2_priv *priv = hr_dev->priv; - struct hnae3_handle *handle = priv->handle; const struct hnae3_ae_ops *ops = handle->ae_algo->ops; - unsigned long instance_stage; - unsigned long reset_cnt; unsigned long end; - bool sw_resetting; - bool hw_resetting; - instance_stage = handle->rinfo.instance_state; - reset_cnt = ops->ae_dev_reset_cnt(handle); - hw_resetting = ops->get_hw_reset_stat(handle); - sw_resetting = ops->ae_dev_resetting(handle); + hr_dev->dis_db = true; + + dev_warn(hr_dev->dev, + "Func clear is pending, device in resetting state.\n"); + end = HNS_ROCE_V2_HW_RST_TIMEOUT; + while (end) { + if (ops->ae_dev_reset_cnt(handle) != + hr_dev->reset_cnt) { + hr_dev->is_reset = true; + dev_info(hr_dev->dev, + "Func clear success after sw reset\n"); + return; + } + msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT); + end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT; + } + + dev_warn(hr_dev->dev, "Func clear failed because of unfinished sw reset\n"); +} + +static void hns_roce_func_clr_rst_proc(struct hns_roce_dev *hr_dev, int retval, + int flag) +{ + struct hns_roce_v2_priv *priv = hr_dev->priv; + struct hnae3_handle *handle = priv->handle; + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; - if (reset_cnt != hr_dev->reset_cnt) { + if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt) { hr_dev->dis_db = true; hr_dev->is_reset = true; dev_info(hr_dev->dev, "Func clear success after reset.\n"); - } else if (hw_resetting) { - hr_dev->dis_db = true; + return; + } - dev_warn(hr_dev->dev, - "Func clear is pending, device in resetting state.\n"); - end = HNS_ROCE_V2_HW_RST_TIMEOUT; - while (end) { - if (!ops->get_hw_reset_stat(handle)) { - hr_dev->is_reset = true; - dev_info(hr_dev->dev, - "Func clear success after reset.\n"); - return; - } - msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT); - end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT; - } + if (ops->get_hw_reset_stat(handle)) { + func_clr_hw_resetting_state(hr_dev, handle); + return; + } - dev_warn(hr_dev->dev, "Func clear failed.\n"); - } else if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT) { - hr_dev->dis_db = true; + if (ops->ae_dev_resetting(handle) && + handle->rinfo.instance_state == HNS_ROCE_STATE_INIT) { + func_clr_sw_resetting_state(hr_dev, handle); + return; + } + if (retval && !flag) dev_warn(hr_dev->dev, - "Func clear is pending, device in resetting state.\n"); - end = HNS_ROCE_V2_HW_RST_TIMEOUT; - while (end) { - if (ops->ae_dev_reset_cnt(handle) != - hr_dev->reset_cnt) { - hr_dev->is_reset = true; - dev_info(hr_dev->dev, - "Func clear success after sw reset\n"); - return; - } - msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT); - end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT; - } - - dev_warn(hr_dev->dev, "Func clear failed because of unfinished sw reset\n"); - } else { - if (retval && !flag) - dev_warn(hr_dev->dev, - "Func clear read failed, ret = %d.\n", retval); + "Func clear read failed, ret = %d.\n", retval); - dev_warn(hr_dev->dev, "Func clear failed.\n"); - } + dev_warn(hr_dev->dev, "Func clear failed.\n"); } -static void hns_roce_function_clear(struct hns_roce_dev *hr_dev) + +static void __hns_roce_function_clear(struct hns_roce_dev *hr_dev, int vf_id) { bool fclr_write_fail_flag = false; struct hns_roce_func_clear *resp; @@ -1484,11 +1553,12 @@ static void hns_roce_function_clear(struct hns_roce_dev *hr_dev) unsigned long end; int ret = 0; - if (hns_roce_func_clr_chk_rst(hr_dev)) + if (check_device_is_in_reset(hr_dev)) goto out; hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, false); resp = (struct hns_roce_func_clear *)desc.data; + resp->rst_funcid_en = cpu_to_le32(vf_id); ret = hns_roce_cmq_send(hr_dev, &desc, 1); if (ret) { @@ -1501,7 +1571,7 @@ static void hns_roce_function_clear(struct hns_roce_dev *hr_dev) msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL); end = HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS; while (end) { - if (hns_roce_func_clr_chk_rst(hr_dev)) + if (check_device_is_in_reset(hr_dev)) goto out; msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT); end -= HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT; @@ -1509,18 +1579,45 @@ static void hns_roce_function_clear(struct hns_roce_dev *hr_dev) hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, true); + resp->rst_funcid_en = cpu_to_le32(vf_id); ret = hns_roce_cmq_send(hr_dev, &desc, 1); if (ret) continue; if (roce_get_bit(resp->func_done, FUNC_CLEAR_RST_FUN_DONE_S)) { - hr_dev->is_reset = true; + if (vf_id == 0) + hr_dev->is_reset = true; return; } } out: - hns_roce_func_clr_rst_prc(hr_dev, ret, fclr_write_fail_flag); + hns_roce_func_clr_rst_proc(hr_dev, ret, fclr_write_fail_flag); +} + +static void hns_roce_free_vf_resource(struct hns_roce_dev *hr_dev, int vf_id) +{ + enum hns_roce_opcode_type opcode = HNS_ROCE_OPC_ALLOC_VF_RES; + struct hns_roce_cmq_desc desc[2]; + struct hns_roce_cmq_req *req_a; + + req_a = (struct hns_roce_cmq_req *)desc[0].data; + hns_roce_cmq_setup_basic_desc(&desc[0], opcode, false); + desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); + hns_roce_cmq_setup_basic_desc(&desc[1], opcode, false); + hr_reg_write(req_a, FUNC_RES_A_VF_ID, vf_id); + hns_roce_cmq_send(hr_dev, desc, 2); +} + +static void hns_roce_function_clear(struct hns_roce_dev *hr_dev) +{ + int i; + + for (i = hr_dev->func_num - 1; i >= 0; i--) { + __hns_roce_function_clear(hr_dev, i); + if (i != 0) + hns_roce_free_vf_resource(hr_dev, i); + } } static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev) @@ -1540,79 +1637,107 @@ static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev) return 0; } +static int hns_roce_query_func_info(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_cmq_desc desc; + int ret; + + if (hr_dev->pci_dev->revision < PCI_REVISION_ID_HIP09) { + hr_dev->func_num = 1; + return 0; + } + + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_FUNC_INFO, + true); + ret = hns_roce_cmq_send(hr_dev, &desc, 1); + if (ret) { + hr_dev->func_num = 1; + return ret; + } + + hr_dev->func_num = le32_to_cpu(desc.func_info.own_func_num); + hr_dev->cong_algo_tmpl_id = le32_to_cpu(desc.func_info.own_mac_id); + + return 0; +} + static int hns_roce_config_global_param(struct hns_roce_dev *hr_dev) { - struct hns_roce_cfg_global_param *req; struct hns_roce_cmq_desc desc; + struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data; hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GLOBAL_PARAM, false); - req = (struct hns_roce_cfg_global_param *)desc.data; - memset(req, 0, sizeof(*req)); - roce_set_field(req->time_cfg_udp_port, - CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_M, - CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_S, 0x3e8); - roce_set_field(req->time_cfg_udp_port, - CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_M, - CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_S, - ROCE_V2_UDP_DPORT); + hr_reg_write(req, CFG_GLOBAL_PARAM_1US_CYCLES, 0x3e8); + hr_reg_write(req, CFG_GLOBAL_PARAM_UDP_PORT, ROCE_V2_UDP_DPORT); return hns_roce_cmq_send(hr_dev, &desc, 1); } -static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev) +static int load_func_res_caps(struct hns_roce_dev *hr_dev, bool is_vf) { struct hns_roce_cmq_desc desc[2]; - struct hns_roce_pf_res_a *req_a; - struct hns_roce_pf_res_b *req_b; + struct hns_roce_cmq_req *r_a = (struct hns_roce_cmq_req *)desc[0].data; + struct hns_roce_cmq_req *r_b = (struct hns_roce_cmq_req *)desc[1].data; + struct hns_roce_caps *caps = &hr_dev->caps; + enum hns_roce_opcode_type opcode; + u32 func_num; int ret; - hns_roce_cmq_setup_basic_desc(&desc[0], HNS_ROCE_OPC_QUERY_PF_RES, - true); - desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); + if (is_vf) { + opcode = HNS_ROCE_OPC_QUERY_VF_RES; + func_num = 1; + } else { + opcode = HNS_ROCE_OPC_QUERY_PF_RES; + func_num = hr_dev->func_num; + } - hns_roce_cmq_setup_basic_desc(&desc[1], HNS_ROCE_OPC_QUERY_PF_RES, - true); + hns_roce_cmq_setup_basic_desc(&desc[0], opcode, true); + desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); + hns_roce_cmq_setup_basic_desc(&desc[1], opcode, true); ret = hns_roce_cmq_send(hr_dev, desc, 2); if (ret) return ret; - req_a = (struct hns_roce_pf_res_a *)desc[0].data; - req_b = (struct hns_roce_pf_res_b *)desc[1].data; - - hr_dev->caps.qpc_bt_num = roce_get_field(req_a->qpc_bt_idx_num, - PF_RES_DATA_1_PF_QPC_BT_NUM_M, - PF_RES_DATA_1_PF_QPC_BT_NUM_S); - hr_dev->caps.srqc_bt_num = roce_get_field(req_a->srqc_bt_idx_num, - PF_RES_DATA_2_PF_SRQC_BT_NUM_M, - PF_RES_DATA_2_PF_SRQC_BT_NUM_S); - hr_dev->caps.cqc_bt_num = roce_get_field(req_a->cqc_bt_idx_num, - PF_RES_DATA_3_PF_CQC_BT_NUM_M, - PF_RES_DATA_3_PF_CQC_BT_NUM_S); - hr_dev->caps.mpt_bt_num = roce_get_field(req_a->mpt_bt_idx_num, - PF_RES_DATA_4_PF_MPT_BT_NUM_M, - PF_RES_DATA_4_PF_MPT_BT_NUM_S); - - hr_dev->caps.sl_num = roce_get_field(req_b->qid_idx_sl_num, - PF_RES_DATA_3_PF_SL_NUM_M, - PF_RES_DATA_3_PF_SL_NUM_S); - hr_dev->caps.sccc_bt_num = roce_get_field(req_b->sccc_bt_idx_num, - PF_RES_DATA_4_PF_SCCC_BT_NUM_M, - PF_RES_DATA_4_PF_SCCC_BT_NUM_S); - - hr_dev->caps.gmv_bt_num = roce_get_field(req_b->gmv_idx_num, - PF_RES_DATA_5_PF_GMV_BT_NUM_M, - PF_RES_DATA_5_PF_GMV_BT_NUM_S); + caps->qpc_bt_num = hr_reg_read(r_a, FUNC_RES_A_QPC_BT_NUM) / func_num; + caps->srqc_bt_num = hr_reg_read(r_a, FUNC_RES_A_SRQC_BT_NUM) / func_num; + caps->cqc_bt_num = hr_reg_read(r_a, FUNC_RES_A_CQC_BT_NUM) / func_num; + caps->mpt_bt_num = hr_reg_read(r_a, FUNC_RES_A_MPT_BT_NUM) / func_num; + caps->eqc_bt_num = hr_reg_read(r_a, FUNC_RES_A_EQC_BT_NUM) / func_num; + caps->smac_bt_num = hr_reg_read(r_b, FUNC_RES_B_SMAC_NUM) / func_num; + caps->sgid_bt_num = hr_reg_read(r_b, FUNC_RES_B_SGID_NUM) / func_num; + caps->sccc_bt_num = hr_reg_read(r_b, FUNC_RES_B_SCCC_BT_NUM) / func_num; + + if (is_vf) { + caps->sl_num = hr_reg_read(r_b, FUNC_RES_V_QID_NUM) / func_num; + caps->gmv_bt_num = hr_reg_read(r_b, FUNC_RES_V_GMV_BT_NUM) / + func_num; + } else { + caps->sl_num = hr_reg_read(r_b, FUNC_RES_B_QID_NUM) / func_num; + caps->gmv_bt_num = hr_reg_read(r_b, FUNC_RES_B_GMV_BT_NUM) / + func_num; + } return 0; } +static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev) +{ + return load_func_res_caps(hr_dev, false); +} + +static int hns_roce_query_vf_resource(struct hns_roce_dev *hr_dev) +{ + return load_func_res_caps(hr_dev, true); +} + static int hns_roce_query_pf_timer_resource(struct hns_roce_dev *hr_dev) { - struct hns_roce_pf_timer_res_a *req_a; struct hns_roce_cmq_desc desc; + struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data; + struct hns_roce_caps *caps = &hr_dev->caps; int ret; hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_PF_TIMER_RES, @@ -1622,24 +1747,17 @@ static int hns_roce_query_pf_timer_resource(struct hns_roce_dev *hr_dev) if (ret) return ret; - req_a = (struct hns_roce_pf_timer_res_a *)desc.data; - - hr_dev->caps.qpc_timer_bt_num = - roce_get_field(req_a->qpc_timer_bt_idx_num, - PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_M, - PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_S); - hr_dev->caps.cqc_timer_bt_num = - roce_get_field(req_a->cqc_timer_bt_idx_num, - PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_M, - PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_S); + caps->qpc_timer_bt_num = hr_reg_read(req, PF_TIMER_RES_QPC_ITEM_NUM); + caps->cqc_timer_bt_num = hr_reg_read(req, PF_TIMER_RES_CQC_ITEM_NUM); return 0; } -static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev, int vf_id) +static int __hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev, + u32 vf_id) { - struct hns_roce_cmq_desc desc; struct hns_roce_vf_switch *swt; + struct hns_roce_cmq_desc desc; int ret; swt = (struct hns_roce_vf_switch *)desc.data; @@ -1661,153 +1779,127 @@ static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev, int vf_id) return hns_roce_cmq_send(hr_dev, &desc, 1); } -static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev) +static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev) { - struct hns_roce_cmq_desc desc[2]; - struct hns_roce_vf_res_a *req_a; - struct hns_roce_vf_res_b *req_b; + u32 vf_id; + int ret; + + for (vf_id = 0; vf_id < hr_dev->func_num; vf_id++) { + ret = __hns_roce_set_vf_switch_param(hr_dev, vf_id); + if (ret) + return ret; + } + return 0; +} - req_a = (struct hns_roce_vf_res_a *)desc[0].data; - req_b = (struct hns_roce_vf_res_b *)desc[1].data; +static int __hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev, int vf_id) +{ + struct hns_roce_cmq_desc desc[2]; + struct hns_roce_cmq_req *r_a = (struct hns_roce_cmq_req *)desc[0].data; + struct hns_roce_cmq_req *r_b = (struct hns_roce_cmq_req *)desc[1].data; + enum hns_roce_opcode_type opcode = HNS_ROCE_OPC_ALLOC_VF_RES; + struct hns_roce_caps *caps = &hr_dev->caps; - hns_roce_cmq_setup_basic_desc(&desc[0], HNS_ROCE_OPC_ALLOC_VF_RES, - false); + hns_roce_cmq_setup_basic_desc(&desc[0], opcode, false); desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); + hns_roce_cmq_setup_basic_desc(&desc[1], opcode, false); - hns_roce_cmq_setup_basic_desc(&desc[1], HNS_ROCE_OPC_ALLOC_VF_RES, - false); + hr_reg_write(r_a, FUNC_RES_A_VF_ID, vf_id); + + hr_reg_write(r_a, FUNC_RES_A_QPC_BT_NUM, caps->qpc_bt_num); + hr_reg_write(r_a, FUNC_RES_A_QPC_BT_IDX, vf_id * caps->qpc_bt_num); + hr_reg_write(r_a, FUNC_RES_A_SRQC_BT_NUM, caps->srqc_bt_num); + hr_reg_write(r_a, FUNC_RES_A_SRQC_BT_IDX, vf_id * caps->srqc_bt_num); + hr_reg_write(r_a, FUNC_RES_A_CQC_BT_NUM, caps->cqc_bt_num); + hr_reg_write(r_a, FUNC_RES_A_CQC_BT_IDX, vf_id * caps->cqc_bt_num); + hr_reg_write(r_a, FUNC_RES_A_MPT_BT_NUM, caps->mpt_bt_num); + hr_reg_write(r_a, FUNC_RES_A_MPT_BT_IDX, vf_id * caps->mpt_bt_num); + hr_reg_write(r_a, FUNC_RES_A_EQC_BT_NUM, caps->eqc_bt_num); + hr_reg_write(r_a, FUNC_RES_A_EQC_BT_IDX, vf_id * caps->eqc_bt_num); + hr_reg_write(r_b, FUNC_RES_V_QID_NUM, caps->sl_num); + hr_reg_write(r_b, FUNC_RES_B_QID_IDX, vf_id * caps->sl_num); + hr_reg_write(r_b, FUNC_RES_B_SCCC_BT_NUM, caps->sccc_bt_num); + hr_reg_write(r_b, FUNC_RES_B_SCCC_BT_IDX, vf_id * caps->sccc_bt_num); - roce_set_field(req_a->vf_qpc_bt_idx_num, - VF_RES_A_DATA_1_VF_QPC_BT_IDX_M, - VF_RES_A_DATA_1_VF_QPC_BT_IDX_S, 0); - roce_set_field(req_a->vf_qpc_bt_idx_num, - VF_RES_A_DATA_1_VF_QPC_BT_NUM_M, - VF_RES_A_DATA_1_VF_QPC_BT_NUM_S, HNS_ROCE_VF_QPC_BT_NUM); - - roce_set_field(req_a->vf_srqc_bt_idx_num, - VF_RES_A_DATA_2_VF_SRQC_BT_IDX_M, - VF_RES_A_DATA_2_VF_SRQC_BT_IDX_S, 0); - roce_set_field(req_a->vf_srqc_bt_idx_num, - VF_RES_A_DATA_2_VF_SRQC_BT_NUM_M, - VF_RES_A_DATA_2_VF_SRQC_BT_NUM_S, - HNS_ROCE_VF_SRQC_BT_NUM); - - roce_set_field(req_a->vf_cqc_bt_idx_num, - VF_RES_A_DATA_3_VF_CQC_BT_IDX_M, - VF_RES_A_DATA_3_VF_CQC_BT_IDX_S, 0); - roce_set_field(req_a->vf_cqc_bt_idx_num, - VF_RES_A_DATA_3_VF_CQC_BT_NUM_M, - VF_RES_A_DATA_3_VF_CQC_BT_NUM_S, HNS_ROCE_VF_CQC_BT_NUM); - - roce_set_field(req_a->vf_mpt_bt_idx_num, - VF_RES_A_DATA_4_VF_MPT_BT_IDX_M, - VF_RES_A_DATA_4_VF_MPT_BT_IDX_S, 0); - roce_set_field(req_a->vf_mpt_bt_idx_num, - VF_RES_A_DATA_4_VF_MPT_BT_NUM_M, - VF_RES_A_DATA_4_VF_MPT_BT_NUM_S, HNS_ROCE_VF_MPT_BT_NUM); - - roce_set_field(req_a->vf_eqc_bt_idx_num, VF_RES_A_DATA_5_VF_EQC_IDX_M, - VF_RES_A_DATA_5_VF_EQC_IDX_S, 0); - roce_set_field(req_a->vf_eqc_bt_idx_num, VF_RES_A_DATA_5_VF_EQC_NUM_M, - VF_RES_A_DATA_5_VF_EQC_NUM_S, HNS_ROCE_VF_EQC_NUM); - - roce_set_field(req_b->vf_smac_idx_num, VF_RES_B_DATA_1_VF_SMAC_IDX_M, - VF_RES_B_DATA_1_VF_SMAC_IDX_S, 0); - roce_set_field(req_b->vf_smac_idx_num, VF_RES_B_DATA_1_VF_SMAC_NUM_M, - VF_RES_B_DATA_1_VF_SMAC_NUM_S, HNS_ROCE_VF_SMAC_NUM); - - roce_set_field(req_b->vf_sgid_idx_num, VF_RES_B_DATA_2_VF_SGID_IDX_M, - VF_RES_B_DATA_2_VF_SGID_IDX_S, 0); - roce_set_field(req_b->vf_sgid_idx_num, VF_RES_B_DATA_2_VF_SGID_NUM_M, - VF_RES_B_DATA_2_VF_SGID_NUM_S, HNS_ROCE_VF_SGID_NUM); - - roce_set_field(req_b->vf_qid_idx_sl_num, VF_RES_B_DATA_3_VF_QID_IDX_M, - VF_RES_B_DATA_3_VF_QID_IDX_S, 0); - roce_set_field(req_b->vf_qid_idx_sl_num, VF_RES_B_DATA_3_VF_SL_NUM_M, - VF_RES_B_DATA_3_VF_SL_NUM_S, HNS_ROCE_VF_SL_NUM); - - roce_set_field(req_b->vf_sccc_idx_num, VF_RES_B_DATA_4_VF_SCCC_BT_IDX_M, - VF_RES_B_DATA_4_VF_SCCC_BT_IDX_S, 0); - roce_set_field(req_b->vf_sccc_idx_num, VF_RES_B_DATA_4_VF_SCCC_BT_NUM_M, - VF_RES_B_DATA_4_VF_SCCC_BT_NUM_S, - HNS_ROCE_VF_SCCC_BT_NUM); + if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) { + hr_reg_write(r_b, FUNC_RES_V_GMV_BT_NUM, caps->gmv_bt_num); + hr_reg_write(r_b, FUNC_RES_B_GMV_BT_IDX, + vf_id * caps->gmv_bt_num); + } else { + hr_reg_write(r_b, FUNC_RES_B_SGID_NUM, caps->sgid_bt_num); + hr_reg_write(r_b, FUNC_RES_B_SGID_IDX, + vf_id * caps->sgid_bt_num); + hr_reg_write(r_b, FUNC_RES_B_SMAC_NUM, caps->smac_bt_num); + hr_reg_write(r_b, FUNC_RES_B_SMAC_IDX, + vf_id * caps->smac_bt_num); + } return hns_roce_cmq_send(hr_dev, desc, 2); } +static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev) +{ + int vf_id; + int ret; + + for (vf_id = 0; vf_id < hr_dev->func_num; vf_id++) { + ret = __hns_roce_alloc_vf_resource(hr_dev, vf_id); + if (ret) + return ret; + } + + return 0; +} + static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev) { - u8 srqc_hop_num = hr_dev->caps.srqc_hop_num; - u8 qpc_hop_num = hr_dev->caps.qpc_hop_num; - u8 cqc_hop_num = hr_dev->caps.cqc_hop_num; - u8 mpt_hop_num = hr_dev->caps.mpt_hop_num; - u8 sccc_hop_num = hr_dev->caps.sccc_hop_num; - struct hns_roce_cfg_bt_attr *req; struct hns_roce_cmq_desc desc; + struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data; + struct hns_roce_caps *caps = &hr_dev->caps; hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_BT_ATTR, false); - req = (struct hns_roce_cfg_bt_attr *)desc.data; - memset(req, 0, sizeof(*req)); - - roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_M, - CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_S, - hr_dev->caps.qpc_ba_pg_sz + PG_SHIFT_OFFSET); - roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_M, - CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_S, - hr_dev->caps.qpc_buf_pg_sz + PG_SHIFT_OFFSET); - roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_M, - CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_S, - qpc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : qpc_hop_num); - - roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_M, - CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_S, - hr_dev->caps.srqc_ba_pg_sz + PG_SHIFT_OFFSET); - roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_M, - CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_S, - hr_dev->caps.srqc_buf_pg_sz + PG_SHIFT_OFFSET); - roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_M, - CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_S, - srqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : srqc_hop_num); - - roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_M, - CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_S, - hr_dev->caps.cqc_ba_pg_sz + PG_SHIFT_OFFSET); - roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_M, - CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_S, - hr_dev->caps.cqc_buf_pg_sz + PG_SHIFT_OFFSET); - roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_M, - CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_S, - cqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : cqc_hop_num); - - roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_M, - CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_S, - hr_dev->caps.mpt_ba_pg_sz + PG_SHIFT_OFFSET); - roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_M, - CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_S, - hr_dev->caps.mpt_buf_pg_sz + PG_SHIFT_OFFSET); - roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_M, - CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_S, - mpt_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : mpt_hop_num); - - roce_set_field(req->vf_sccc_cfg, - CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_M, - CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_S, - hr_dev->caps.sccc_ba_pg_sz + PG_SHIFT_OFFSET); - roce_set_field(req->vf_sccc_cfg, - CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_M, - CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_S, - hr_dev->caps.sccc_buf_pg_sz + PG_SHIFT_OFFSET); - roce_set_field(req->vf_sccc_cfg, - CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_M, - CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_S, - sccc_hop_num == - HNS_ROCE_HOP_NUM_0 ? 0 : sccc_hop_num); + + hr_reg_write(req, CFG_BT_ATTR_QPC_BA_PGSZ, + caps->qpc_ba_pg_sz + PG_SHIFT_OFFSET); + hr_reg_write(req, CFG_BT_ATTR_QPC_BUF_PGSZ, + caps->qpc_buf_pg_sz + PG_SHIFT_OFFSET); + hr_reg_write(req, CFG_BT_ATTR_QPC_HOPNUM, + to_hr_hem_hopnum(caps->qpc_hop_num, caps->num_qps)); + + hr_reg_write(req, CFG_BT_ATTR_SRQC_BA_PGSZ, + caps->srqc_ba_pg_sz + PG_SHIFT_OFFSET); + hr_reg_write(req, CFG_BT_ATTR_SRQC_BUF_PGSZ, + caps->srqc_buf_pg_sz + PG_SHIFT_OFFSET); + hr_reg_write(req, CFG_BT_ATTR_SRQC_HOPNUM, + to_hr_hem_hopnum(caps->srqc_hop_num, caps->num_srqs)); + + hr_reg_write(req, CFG_BT_ATTR_CQC_BA_PGSZ, + caps->cqc_ba_pg_sz + PG_SHIFT_OFFSET); + hr_reg_write(req, CFG_BT_ATTR_CQC_BUF_PGSZ, + caps->cqc_buf_pg_sz + PG_SHIFT_OFFSET); + hr_reg_write(req, CFG_BT_ATTR_CQC_HOPNUM, + to_hr_hem_hopnum(caps->cqc_hop_num, caps->num_cqs)); + + hr_reg_write(req, CFG_BT_ATTR_MPT_BA_PGSZ, + caps->mpt_ba_pg_sz + PG_SHIFT_OFFSET); + hr_reg_write(req, CFG_BT_ATTR_MPT_BUF_PGSZ, + caps->mpt_buf_pg_sz + PG_SHIFT_OFFSET); + hr_reg_write(req, CFG_BT_ATTR_MPT_HOPNUM, + to_hr_hem_hopnum(caps->mpt_hop_num, caps->num_mtpts)); + + hr_reg_write(req, CFG_BT_ATTR_SCCC_BA_PGSZ, + caps->sccc_ba_pg_sz + PG_SHIFT_OFFSET); + hr_reg_write(req, CFG_BT_ATTR_SCCC_BUF_PGSZ, + caps->sccc_buf_pg_sz + PG_SHIFT_OFFSET); + hr_reg_write(req, CFG_BT_ATTR_SCCC_HOPNUM, + to_hr_hem_hopnum(caps->sccc_hop_num, caps->num_qps)); return hns_roce_cmq_send(hr_dev, &desc, 1); } static void set_default_caps(struct hns_roce_dev *hr_dev) { + struct hns_roce_v2_priv *priv = hr_dev->priv; struct hns_roce_caps *caps = &hr_dev->caps; caps->num_qps = HNS_ROCE_V2_MAX_QP_NUM; @@ -1819,24 +1911,24 @@ static void set_default_caps(struct hns_roce_dev *hr_dev) caps->max_sq_sg = HNS_ROCE_V2_MAX_SQ_SGE_NUM; caps->max_extend_sg = HNS_ROCE_V2_MAX_EXTEND_SGE_NUM; caps->max_rq_sg = HNS_ROCE_V2_MAX_RQ_SGE_NUM; - caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INLINE; caps->num_uars = HNS_ROCE_V2_UAR_NUM; caps->phy_num_uars = HNS_ROCE_V2_PHY_UAR_NUM; caps->num_aeq_vectors = HNS_ROCE_V2_AEQE_VEC_NUM; - caps->num_comp_vectors = HNS_ROCE_V2_COMP_VEC_NUM; + caps->num_comp_vectors = + min_t(u32, caps->eqc_bt_num - 1, + (u32)priv->handle->rinfo.num_vectors - 2); caps->num_other_vectors = HNS_ROCE_V2_ABNORMAL_VEC_NUM; caps->num_mtpts = HNS_ROCE_V2_MAX_MTPT_NUM; caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS; - caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS; caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS; caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS; caps->num_pds = HNS_ROCE_V2_MAX_PD_NUM; + caps->num_xrcds = HNS_ROCE_V2_MAX_XRCD_NUM; caps->max_qp_init_rdma = HNS_ROCE_V2_MAX_QP_INIT_RDMA; caps->max_qp_dest_rdma = HNS_ROCE_V2_MAX_QP_DEST_RDMA; caps->max_sq_desc_sz = HNS_ROCE_V2_MAX_SQ_DESC_SZ; caps->max_rq_desc_sz = HNS_ROCE_V2_MAX_RQ_DESC_SZ; caps->max_srq_desc_sz = HNS_ROCE_V2_MAX_SRQ_DESC_SZ; - caps->qpc_sz = HNS_ROCE_V2_QPC_SZ; caps->irrl_entry_sz = HNS_ROCE_V2_IRRL_ENTRY_SZ; caps->trrl_entry_sz = HNS_ROCE_V2_EXT_ATOMIC_TRRL_ENTRY_SZ; caps->cqc_entry_sz = HNS_ROCE_V2_CQC_ENTRY_SZ; @@ -1844,56 +1936,39 @@ static void set_default_caps(struct hns_roce_dev *hr_dev) caps->mtpt_entry_sz = HNS_ROCE_V2_MTPT_ENTRY_SZ; caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ; caps->idx_entry_sz = HNS_ROCE_V2_IDX_ENTRY_SZ; - caps->cqe_sz = HNS_ROCE_V2_CQE_SIZE; caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED; caps->reserved_lkey = 0; caps->reserved_pds = 0; + caps->reserved_xrcds = HNS_ROCE_V2_RSV_XRCD_NUM; caps->reserved_mrws = 1; caps->reserved_uars = 0; caps->reserved_cqs = 0; caps->reserved_srqs = 0; caps->reserved_qps = HNS_ROCE_V2_RSV_QPS; - caps->qpc_ba_pg_sz = 0; - caps->qpc_buf_pg_sz = 0; caps->qpc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; - caps->srqc_ba_pg_sz = 0; - caps->srqc_buf_pg_sz = 0; caps->srqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; - caps->cqc_ba_pg_sz = 0; - caps->cqc_buf_pg_sz = 0; caps->cqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; - caps->mpt_ba_pg_sz = 0; - caps->mpt_buf_pg_sz = 0; caps->mpt_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; - caps->mtt_ba_pg_sz = 0; - caps->mtt_buf_pg_sz = 0; caps->mtt_hop_num = HNS_ROCE_MTT_HOP_NUM; + caps->pbl_hop_num = HNS_ROCE_PBL_HOP_NUM; caps->wqe_sq_hop_num = HNS_ROCE_SQWQE_HOP_NUM; caps->wqe_sge_hop_num = HNS_ROCE_EXT_SGE_HOP_NUM; caps->wqe_rq_hop_num = HNS_ROCE_RQWQE_HOP_NUM; - caps->cqe_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_256K; - caps->cqe_buf_pg_sz = 0; caps->cqe_hop_num = HNS_ROCE_CQE_HOP_NUM; - caps->srqwqe_ba_pg_sz = 0; - caps->srqwqe_buf_pg_sz = 0; caps->srqwqe_hop_num = HNS_ROCE_SRQWQE_HOP_NUM; - caps->idx_ba_pg_sz = 0; - caps->idx_buf_pg_sz = 0; caps->idx_hop_num = HNS_ROCE_IDX_HOP_NUM; - caps->chunk_sz = HNS_ROCE_V2_TABLE_CHUNK_SIZE; + caps->eqe_hop_num = HNS_ROCE_EQE_HOP_NUM; + caps->chunk_sz = HNS_ROCE_V2_TABLE_CHUNK_SIZE; caps->flags = HNS_ROCE_CAP_FLAG_REREG_MR | HNS_ROCE_CAP_FLAG_ROCE_V1_V2 | - HNS_ROCE_CAP_FLAG_RECORD_DB | - HNS_ROCE_CAP_FLAG_SQ_RECORD_DB; + HNS_ROCE_CAP_FLAG_CQ_RECORD_DB | + HNS_ROCE_CAP_FLAG_QP_RECORD_DB; caps->pkey_table_len[0] = 1; - caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM; caps->ceqe_depth = HNS_ROCE_V2_COMP_EQE_NUM; caps->aeqe_depth = HNS_ROCE_V2_ASYNC_EQE_NUM; - caps->aeqe_size = HNS_ROCE_AEQE_SIZE; - caps->ceqe_size = HNS_ROCE_CEQE_SIZE; caps->local_ca_ack_delay = 0; caps->max_mtu = IB_MTU_4096; @@ -1902,22 +1977,15 @@ static void set_default_caps(struct hns_roce_dev *hr_dev) caps->flags |= HNS_ROCE_CAP_FLAG_ATOMIC | HNS_ROCE_CAP_FLAG_MW | HNS_ROCE_CAP_FLAG_SRQ | HNS_ROCE_CAP_FLAG_FRMR | - HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL; + HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL | HNS_ROCE_CAP_FLAG_XRC; caps->num_qpc_timer = HNS_ROCE_V2_MAX_QPC_TIMER_NUM; caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ; - caps->qpc_timer_ba_pg_sz = 0; - caps->qpc_timer_buf_pg_sz = 0; caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0; caps->num_cqc_timer = HNS_ROCE_V2_MAX_CQC_TIMER_NUM; caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ; - caps->cqc_timer_ba_pg_sz = 0; - caps->cqc_timer_buf_pg_sz = 0; caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0; - caps->sccc_sz = HNS_ROCE_V2_SCCC_SZ; - caps->sccc_ba_pg_sz = 0; - caps->sccc_buf_pg_sz = 0; caps->sccc_hop_num = HNS_ROCE_SCCC_HOP_NUM; if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) { @@ -1930,10 +1998,17 @@ static void set_default_caps(struct hns_roce_dev *hr_dev) caps->gmv_entry_num = caps->gmv_bt_num * (PAGE_SIZE / caps->gmv_entry_sz); caps->gmv_hop_num = HNS_ROCE_HOP_NUM_0; - caps->gmv_ba_pg_sz = 0; - caps->gmv_buf_pg_sz = 0; caps->gid_table_len[0] = caps->gmv_bt_num * (HNS_HW_PAGE_SIZE / caps->gmv_entry_sz); + caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INL_EXT; + } else { + caps->aeqe_size = HNS_ROCE_AEQE_SIZE; + caps->ceqe_size = HNS_ROCE_CEQE_SIZE; + caps->cqe_sz = HNS_ROCE_V2_CQE_SIZE; + caps->qpc_sz = HNS_ROCE_V2_QPC_SZ; + caps->sccc_sz = HNS_ROCE_V2_SCCC_SZ; + caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM; + caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INLINE; } } @@ -1979,6 +2054,70 @@ static void calc_pg_sz(u32 obj_num, u32 obj_size, u32 hop_num, u32 ctx_bt_num, *buf_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk)); } +static void set_hem_page_size(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_caps *caps = &hr_dev->caps; + + /* EQ */ + caps->eqe_ba_pg_sz = 0; + caps->eqe_buf_pg_sz = 0; + + /* Link Table */ + caps->tsq_buf_pg_sz = 0; + + /* MR */ + caps->pbl_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_16K; + caps->pbl_buf_pg_sz = 0; + calc_pg_sz(caps->num_mtpts, caps->mtpt_entry_sz, caps->mpt_hop_num, + caps->mpt_bt_num, &caps->mpt_buf_pg_sz, &caps->mpt_ba_pg_sz, + HEM_TYPE_MTPT); + + /* QP */ + caps->qpc_timer_ba_pg_sz = 0; + caps->qpc_timer_buf_pg_sz = 0; + caps->mtt_ba_pg_sz = 0; + caps->mtt_buf_pg_sz = 0; + calc_pg_sz(caps->num_qps, caps->qpc_sz, caps->qpc_hop_num, + caps->qpc_bt_num, &caps->qpc_buf_pg_sz, &caps->qpc_ba_pg_sz, + HEM_TYPE_QPC); + + if (caps->flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) + calc_pg_sz(caps->num_qps, caps->sccc_sz, caps->sccc_hop_num, + caps->sccc_bt_num, &caps->sccc_buf_pg_sz, + &caps->sccc_ba_pg_sz, HEM_TYPE_SCCC); + + /* CQ */ + calc_pg_sz(caps->num_cqs, caps->cqc_entry_sz, caps->cqc_hop_num, + caps->cqc_bt_num, &caps->cqc_buf_pg_sz, &caps->cqc_ba_pg_sz, + HEM_TYPE_CQC); + calc_pg_sz(caps->max_cqes, caps->cqe_sz, caps->cqe_hop_num, + 1, &caps->cqe_buf_pg_sz, &caps->cqe_ba_pg_sz, HEM_TYPE_CQE); + + if (caps->cqc_timer_entry_sz) + calc_pg_sz(caps->num_cqc_timer, caps->cqc_timer_entry_sz, + caps->cqc_timer_hop_num, caps->cqc_timer_bt_num, + &caps->cqc_timer_buf_pg_sz, + &caps->cqc_timer_ba_pg_sz, HEM_TYPE_CQC_TIMER); + + /* SRQ */ + if (caps->flags & HNS_ROCE_CAP_FLAG_SRQ) { + calc_pg_sz(caps->num_srqs, caps->srqc_entry_sz, + caps->srqc_hop_num, caps->srqc_bt_num, + &caps->srqc_buf_pg_sz, &caps->srqc_ba_pg_sz, + HEM_TYPE_SRQC); + calc_pg_sz(caps->num_srqwqe_segs, caps->mtt_entry_sz, + caps->srqwqe_hop_num, 1, &caps->srqwqe_buf_pg_sz, + &caps->srqwqe_ba_pg_sz, HEM_TYPE_SRQWQE); + calc_pg_sz(caps->num_idx_segs, caps->idx_entry_sz, + caps->idx_hop_num, 1, &caps->idx_buf_pg_sz, + &caps->idx_ba_pg_sz, HEM_TYPE_IDX); + } + + /* GMV */ + caps->gmv_ba_pg_sz = 0; + caps->gmv_buf_pg_sz = 0; +} + static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) { struct hns_roce_cmq_desc desc[HNS_ROCE_QUERY_PF_CAPS_CMD_NUM]; @@ -2062,6 +2201,9 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) caps->gid_table_len[0] = roce_get_field(resp_c->max_gid_num_cqs, V2_QUERY_PF_CAPS_C_MAX_GID_M, V2_QUERY_PF_CAPS_C_MAX_GID_S); + + caps->gid_table_len[0] /= hr_dev->func_num; + caps->max_cqes = 1 << roce_get_field(resp_c->cq_depth, V2_QUERY_PF_CAPS_C_CQ_DEPTH_M, V2_QUERY_PF_CAPS_C_CQ_DEPTH_S); @@ -2079,13 +2221,18 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) caps->num_srqs = 1 << roce_get_field(resp_d->wq_hop_num_max_srqs, V2_QUERY_PF_CAPS_D_NUM_SRQS_M, V2_QUERY_PF_CAPS_D_NUM_SRQS_S); + caps->cong_type = roce_get_field(resp_d->wq_hop_num_max_srqs, + V2_QUERY_PF_CAPS_D_CONG_TYPE_M, + V2_QUERY_PF_CAPS_D_CONG_TYPE_S); caps->max_srq_wrs = 1 << le16_to_cpu(resp_d->srq_depth); + caps->ceqe_depth = 1 << roce_get_field(resp_d->num_ceqs_ceq_depth, V2_QUERY_PF_CAPS_D_CEQ_DEPTH_M, V2_QUERY_PF_CAPS_D_CEQ_DEPTH_S); caps->num_comp_vectors = roce_get_field(resp_d->num_ceqs_ceq_depth, V2_QUERY_PF_CAPS_D_NUM_CEQS_M, V2_QUERY_PF_CAPS_D_NUM_CEQS_S); + caps->aeqe_depth = 1 << roce_get_field(resp_d->arm_st_aeq_depth, V2_QUERY_PF_CAPS_D_AEQ_DEPTH_M, V2_QUERY_PF_CAPS_D_AEQ_DEPTH_S); @@ -2133,8 +2280,8 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS; caps->ceqe_size = HNS_ROCE_CEQE_SIZE; caps->aeqe_size = HNS_ROCE_AEQE_SIZE; - caps->mtt_ba_pg_sz = 0; - caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS; + caps->num_xrcds = HNS_ROCE_V2_MAX_XRCD_NUM; + caps->reserved_xrcds = HNS_ROCE_V2_RSV_XRCD_NUM; caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS; caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS; @@ -2166,99 +2313,82 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) caps->gmv_entry_num = caps->gmv_bt_num * (PAGE_SIZE / caps->gmv_entry_sz); caps->gmv_hop_num = HNS_ROCE_HOP_NUM_0; - caps->gmv_ba_pg_sz = 0; - caps->gmv_buf_pg_sz = 0; caps->gid_table_len[0] = caps->gmv_bt_num * (HNS_HW_PAGE_SIZE / caps->gmv_entry_sz); } - calc_pg_sz(caps->num_qps, caps->qpc_sz, caps->qpc_hop_num, - caps->qpc_bt_num, &caps->qpc_buf_pg_sz, &caps->qpc_ba_pg_sz, - HEM_TYPE_QPC); - calc_pg_sz(caps->num_mtpts, caps->mtpt_entry_sz, caps->mpt_hop_num, - caps->mpt_bt_num, &caps->mpt_buf_pg_sz, &caps->mpt_ba_pg_sz, - HEM_TYPE_MTPT); - calc_pg_sz(caps->num_cqs, caps->cqc_entry_sz, caps->cqc_hop_num, - caps->cqc_bt_num, &caps->cqc_buf_pg_sz, &caps->cqc_ba_pg_sz, - HEM_TYPE_CQC); - calc_pg_sz(caps->num_srqs, caps->srqc_entry_sz, caps->srqc_hop_num, - caps->srqc_bt_num, &caps->srqc_buf_pg_sz, - &caps->srqc_ba_pg_sz, HEM_TYPE_SRQC); - - caps->sccc_hop_num = ctx_hop_num; caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0; caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0; - calc_pg_sz(caps->num_qps, caps->sccc_sz, - caps->sccc_hop_num, caps->sccc_bt_num, - &caps->sccc_buf_pg_sz, &caps->sccc_ba_pg_sz, - HEM_TYPE_SCCC); - calc_pg_sz(caps->num_cqc_timer, caps->cqc_timer_entry_sz, - caps->cqc_timer_hop_num, caps->cqc_timer_bt_num, - &caps->cqc_timer_buf_pg_sz, - &caps->cqc_timer_ba_pg_sz, HEM_TYPE_CQC_TIMER); - - calc_pg_sz(caps->num_cqe_segs, caps->mtt_entry_sz, caps->cqe_hop_num, - 1, &caps->cqe_buf_pg_sz, &caps->cqe_ba_pg_sz, HEM_TYPE_CQE); - calc_pg_sz(caps->num_srqwqe_segs, caps->mtt_entry_sz, - caps->srqwqe_hop_num, 1, &caps->srqwqe_buf_pg_sz, - &caps->srqwqe_ba_pg_sz, HEM_TYPE_SRQWQE); - calc_pg_sz(caps->num_idx_segs, caps->idx_entry_sz, caps->idx_hop_num, - 1, &caps->idx_buf_pg_sz, &caps->idx_ba_pg_sz, HEM_TYPE_IDX); - return 0; } -static int hns_roce_config_qpc_size(struct hns_roce_dev *hr_dev) +static int config_hem_entry_size(struct hns_roce_dev *hr_dev, u32 type, u32 val) { struct hns_roce_cmq_desc desc; - struct hns_roce_cfg_entry_size *cfg_size = - (struct hns_roce_cfg_entry_size *)desc.data; + struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data; hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_ENTRY_SIZE, false); - cfg_size->type = cpu_to_le32(HNS_ROCE_CFG_QPC_SIZE); - cfg_size->size = cpu_to_le32(hr_dev->caps.qpc_sz); - - return hns_roce_cmq_send(hr_dev, &desc, 1); -} - -static int hns_roce_config_sccc_size(struct hns_roce_dev *hr_dev) -{ - struct hns_roce_cmq_desc desc; - struct hns_roce_cfg_entry_size *cfg_size = - (struct hns_roce_cfg_entry_size *)desc.data; - - hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_ENTRY_SIZE, - false); - - cfg_size->type = cpu_to_le32(HNS_ROCE_CFG_SCCC_SIZE); - cfg_size->size = cpu_to_le32(hr_dev->caps.sccc_sz); + hr_reg_write(req, CFG_HEM_ENTRY_SIZE_TYPE, type); + hr_reg_write(req, CFG_HEM_ENTRY_SIZE_VALUE, val); return hns_roce_cmq_send(hr_dev, &desc, 1); } static int hns_roce_config_entry_size(struct hns_roce_dev *hr_dev) { + struct hns_roce_caps *caps = &hr_dev->caps; int ret; if (hr_dev->pci_dev->revision < PCI_REVISION_ID_HIP09) return 0; - ret = hns_roce_config_qpc_size(hr_dev); + ret = config_hem_entry_size(hr_dev, HNS_ROCE_CFG_QPC_SIZE, + caps->qpc_sz); if (ret) { dev_err(hr_dev->dev, "failed to cfg qpc sz, ret = %d.\n", ret); return ret; } - ret = hns_roce_config_sccc_size(hr_dev); + ret = config_hem_entry_size(hr_dev, HNS_ROCE_CFG_SCCC_SIZE, + caps->sccc_sz); if (ret) dev_err(hr_dev->dev, "failed to cfg sccc sz, ret = %d.\n", ret); return ret; } +static int hns_roce_v2_vf_profile(struct hns_roce_dev *hr_dev) +{ + int ret; + + hr_dev->vendor_part_id = hr_dev->pci_dev->device; + hr_dev->sys_image_guid = be64_to_cpu(hr_dev->ib_dev.node_guid); + hr_dev->func_num = 1; + + ret = hns_roce_query_vf_resource(hr_dev); + if (ret) { + dev_err(hr_dev->dev, + "Query the VF resource fail, ret = %d.\n", ret); + return ret; + } + + set_default_caps(hr_dev); + set_hem_page_size(hr_dev); + + ret = hns_roce_v2_set_bt(hr_dev); + if (ret) { + dev_err(hr_dev->dev, + "Configure the VF bt attribute fail, ret = %d.\n", + ret); + return ret; + } + + return 0; +} + static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) { struct hns_roce_caps *caps = &hr_dev->caps; @@ -2278,6 +2408,16 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) return ret; } + if (hr_dev->is_vf) + return hns_roce_v2_vf_profile(hr_dev); + + ret = hns_roce_query_func_info(hr_dev); + if (ret) { + dev_err(hr_dev->dev, "Query function info fail, ret = %d.\n", + ret); + return ret; + } + ret = hns_roce_config_global_param(hr_dev); if (ret) { dev_err(hr_dev->dev, "Configure global param fail, ret = %d.\n", @@ -2300,7 +2440,7 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) return ret; } - ret = hns_roce_set_vf_switch_param(hr_dev, 0); + ret = hns_roce_set_vf_switch_param(hr_dev); if (ret) { dev_err(hr_dev->dev, "failed to set function switch param, ret = %d.\n", @@ -2311,13 +2451,8 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) hr_dev->vendor_part_id = hr_dev->pci_dev->device; hr_dev->sys_image_guid = be64_to_cpu(hr_dev->ib_dev.node_guid); - caps->pbl_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_16K; - caps->pbl_buf_pg_sz = 0; caps->pbl_hop_num = HNS_ROCE_PBL_HOP_NUM; - caps->eqe_ba_pg_sz = 0; - caps->eqe_buf_pg_sz = 0; caps->eqe_hop_num = HNS_ROCE_EQE_HOP_NUM; - caps->tsq_buf_pg_sz = 0; ret = hns_roce_query_pf_caps(hr_dev); if (ret) @@ -2330,6 +2465,7 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) return ret; } + set_hem_page_size(hr_dev); ret = hns_roce_v2_set_bt(hr_dev); if (ret) { dev_err(hr_dev->dev, @@ -2507,6 +2643,22 @@ static void hns_roce_free_link_table(struct hns_roce_dev *hr_dev, link_tbl->table.map); } +static void free_dip_list(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_dip *hr_dip; + struct hns_roce_dip *tmp; + unsigned long flags; + + spin_lock_irqsave(&hr_dev->dip_list_lock, flags); + + list_for_each_entry_safe(hr_dip, tmp, &hr_dev->dip_list, node) { + list_del(&hr_dip->node); + kfree(hr_dip); + } + + spin_unlock_irqrestore(&hr_dev->dip_list_lock, flags); +} + static int get_hem_table(struct hns_roce_dev *hr_dev) { unsigned int qpc_count; @@ -2515,6 +2667,17 @@ static int get_hem_table(struct hns_roce_dev *hr_dev) int ret; int i; + /* Alloc memory for source address table buffer space chunk */ + for (gmv_count = 0; gmv_count < hr_dev->caps.gmv_entry_num; + gmv_count++) { + ret = hns_roce_table_get(hr_dev, &hr_dev->gmv_table, gmv_count); + if (ret) + goto err_gmv_failed; + } + + if (hr_dev->is_vf) + return 0; + /* Alloc memory for QPC Timer buffer space chunk */ for (qpc_count = 0; qpc_count < hr_dev->caps.qpc_timer_bt_num; qpc_count++) { @@ -2537,23 +2700,8 @@ static int get_hem_table(struct hns_roce_dev *hr_dev) } } - /* Alloc memory for GMV(GID/MAC/VLAN) table buffer space chunk */ - for (gmv_count = 0; gmv_count < hr_dev->caps.gmv_entry_num; - gmv_count++) { - ret = hns_roce_table_get(hr_dev, &hr_dev->gmv_table, gmv_count); - if (ret) { - dev_err(hr_dev->dev, - "failed to get gmv table, ret = %d.\n", ret); - goto err_gmv_failed; - } - } - return 0; -err_gmv_failed: - for (i = 0; i < gmv_count; i++) - hns_roce_table_put(hr_dev, &hr_dev->gmv_table, i); - err_cqc_timer_failed: for (i = 0; i < cqc_count; i++) hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i); @@ -2562,19 +2710,47 @@ err_qpc_timer_failed: for (i = 0; i < qpc_count; i++) hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i); +err_gmv_failed: + for (i = 0; i < gmv_count; i++) + hns_roce_table_put(hr_dev, &hr_dev->gmv_table, i); + return ret; } +static void put_hem_table(struct hns_roce_dev *hr_dev) +{ + int i; + + for (i = 0; i < hr_dev->caps.gmv_entry_num; i++) + hns_roce_table_put(hr_dev, &hr_dev->gmv_table, i); + + if (hr_dev->is_vf) + return; + + for (i = 0; i < hr_dev->caps.qpc_timer_bt_num; i++) + hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i); + + for (i = 0; i < hr_dev->caps.cqc_timer_bt_num; i++) + hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i); +} + static int hns_roce_v2_init(struct hns_roce_dev *hr_dev) { struct hns_roce_v2_priv *priv = hr_dev->priv; int ret; + ret = get_hem_table(hr_dev); + if (ret) + return ret; + + if (hr_dev->is_vf) + return 0; + /* TSQ includes SQ doorbell and ack doorbell */ ret = hns_roce_init_link_table(hr_dev, TSQ_LINK_TABLE); if (ret) { dev_err(hr_dev->dev, "failed to init TSQ, ret = %d.\n", ret); - return ret; + goto err_tsq_init_failed; } ret = hns_roce_init_link_table(hr_dev, TPQ_LINK_TABLE); @@ -2583,17 +2759,13 @@ static int hns_roce_v2_init(struct hns_roce_dev *hr_dev) goto err_tpq_init_failed; } - ret = get_hem_table(hr_dev); - if (ret) - goto err_get_hem_table_failed; - return 0; -err_get_hem_table_failed: - hns_roce_free_link_table(hr_dev, &priv->tpq); +err_tsq_init_failed: + put_hem_table(hr_dev); err_tpq_init_failed: - hns_roce_free_link_table(hr_dev, &priv->tsq); + hns_roce_free_link_table(hr_dev, &priv->tpq); return ret; } @@ -2604,38 +2776,13 @@ static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev) hns_roce_function_clear(hr_dev); - hns_roce_free_link_table(hr_dev, &priv->tpq); - hns_roce_free_link_table(hr_dev, &priv->tsq); -} - -static int hns_roce_query_mbox_status(struct hns_roce_dev *hr_dev) -{ - struct hns_roce_cmq_desc desc; - struct hns_roce_mbox_status *mb_st = - (struct hns_roce_mbox_status *)desc.data; - int status; - - hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_MB_ST, true); - - status = hns_roce_cmq_send(hr_dev, &desc, 1); - if (status) - return status; - - return le32_to_cpu(mb_st->mb_status_hw_run); -} - -static int hns_roce_v2_cmd_pending(struct hns_roce_dev *hr_dev) -{ - u32 status = hns_roce_query_mbox_status(hr_dev); - - return status >> HNS_ROCE_HW_RUN_BIT_SHIFT; -} - -static int hns_roce_v2_cmd_complete(struct hns_roce_dev *hr_dev) -{ - u32 status = hns_roce_query_mbox_status(hr_dev); + if (!hr_dev->is_vf) { + hns_roce_free_link_table(hr_dev, &priv->tpq); + hns_roce_free_link_table(hr_dev, &priv->tsq); + } - return status & HNS_ROCE_HW_MB_STATUS_MASK; + if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP09) + free_dip_list(hr_dev); } static int hns_roce_mbox_post(struct hns_roce_dev *hr_dev, u64 in_param, @@ -2657,58 +2804,97 @@ static int hns_roce_mbox_post(struct hns_roce_dev *hr_dev, u64 in_param, return hns_roce_cmq_send(hr_dev, &desc, 1); } -static int hns_roce_v2_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param, - u64 out_param, u32 in_modifier, u8 op_modifier, - u16 op, u16 token, int event) +static int v2_wait_mbox_complete(struct hns_roce_dev *hr_dev, u32 timeout, + u8 *complete_status) { - struct device *dev = hr_dev->dev; + struct hns_roce_mbox_status *mb_st; + struct hns_roce_cmq_desc desc; unsigned long end; - int ret; + int ret = -EBUSY; + u32 status; + bool busy; + + mb_st = (struct hns_roce_mbox_status *)desc.data; + end = msecs_to_jiffies(timeout) + jiffies; + while (v2_chk_mbox_is_avail(hr_dev, &busy)) { + status = 0; + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_MB_ST, + true); + ret = __hns_roce_cmq_send(hr_dev, &desc, 1); + if (!ret) { + status = le32_to_cpu(mb_st->mb_status_hw_run); + /* No pending message exists in ROCEE mbox. */ + if (!(status & MB_ST_HW_RUN_M)) + break; + } else if (!v2_chk_mbox_is_avail(hr_dev, &busy)) { + break; + } - end = msecs_to_jiffies(HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS) + jiffies; - while (hns_roce_v2_cmd_pending(hr_dev)) { if (time_after(jiffies, end)) { - dev_dbg(dev, "jiffies=%d end=%d\n", (int)jiffies, - (int)end); - return -EAGAIN; + dev_err_ratelimited(hr_dev->dev, + "failed to wait mbox status 0x%x\n", + status); + return -ETIMEDOUT; } + cond_resched(); + ret = -EBUSY; } - ret = hns_roce_mbox_post(hr_dev, in_param, out_param, in_modifier, - op_modifier, op, token, event); - if (ret) - dev_err(dev, "Post mailbox fail(%d)\n", ret); + if (!ret) { + *complete_status = (u8)(status & MB_ST_COMPLETE_M); + } else if (!v2_chk_mbox_is_avail(hr_dev, &busy)) { + /* Ignore all errors if the mbox is unavailable. */ + ret = 0; + *complete_status = MB_ST_COMPLETE_M; + } return ret; } -static int hns_roce_v2_chk_mbox(struct hns_roce_dev *hr_dev, - unsigned int timeout) +static int v2_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param, + u64 out_param, u32 in_modifier, u8 op_modifier, + u16 op, u16 token, int event) { - struct device *dev = hr_dev->dev; - unsigned long end; - u32 status; - - end = msecs_to_jiffies(timeout) + jiffies; - while (hns_roce_v2_cmd_pending(hr_dev) && time_before(jiffies, end)) - cond_resched(); + u8 status = 0; + int ret; - if (hns_roce_v2_cmd_pending(hr_dev)) { - dev_err(dev, "[cmd_poll]hw run cmd TIMEDOUT!\n"); - return -ETIMEDOUT; + /* Waiting for the mbox to be idle */ + ret = v2_wait_mbox_complete(hr_dev, HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS, + &status); + if (unlikely(ret)) { + dev_err_ratelimited(hr_dev->dev, + "failed to check post mbox status = 0x%x, ret = %d.\n", + status, ret); + return ret; } - status = hns_roce_v2_cmd_complete(hr_dev); - if (status != 0x1) { - if (status == CMD_RST_PRC_EBUSY) - return status; + /* Post new message to mbox */ + ret = hns_roce_mbox_post(hr_dev, in_param, out_param, in_modifier, + op_modifier, op, token, event); + if (ret) + dev_err_ratelimited(hr_dev->dev, + "failed to post mailbox, ret = %d.\n", ret); + + return ret; +} - dev_err(dev, "mailbox status 0x%x!\n", status); - return -EBUSY; +static int v2_poll_mbox_done(struct hns_roce_dev *hr_dev, unsigned int timeout) +{ + u8 status = 0; + int ret; + + ret = v2_wait_mbox_complete(hr_dev, timeout, &status); + if (!ret) { + if (status != MB_ST_COMPLETE_SUCC) + return -EBUSY; + } else { + dev_err_ratelimited(hr_dev->dev, + "failed to check mbox status = 0x%x, ret = %d.\n", + status, ret); } - return 0; + return ret; } static void copy_gid(void *dest, const union ib_gid *gid) @@ -2790,7 +2976,7 @@ static int config_gmv_table(struct hns_roce_dev *hr_dev, return hns_roce_cmq_send(hr_dev, desc, 2); } -static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, u8 port, +static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, u32 port, int gid_index, const union ib_gid *gid, const struct ib_gid_attr *attr) { @@ -3079,14 +3265,31 @@ static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, unsigned int n) !!(n & hr_cq->cq_depth)) ? cqe : NULL; } -static inline void hns_roce_v2_cq_set_ci(struct hns_roce_cq *hr_cq, u32 ci) +static inline void update_cq_db(struct hns_roce_dev *hr_dev, + struct hns_roce_cq *hr_cq) { - *hr_cq->set_ci_db = ci & V2_CQ_DB_PARAMETER_CONS_IDX_M; + if (likely(hr_cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB)) { + *hr_cq->set_ci_db = hr_cq->cons_index & V2_CQ_DB_CONS_IDX_M; + } else { + struct hns_roce_v2_db cq_db = {}; + + roce_set_field(cq_db.byte_4, V2_DB_TAG_M, V2_DB_TAG_S, + hr_cq->cqn); + roce_set_field(cq_db.byte_4, V2_DB_CMD_M, V2_DB_CMD_S, + HNS_ROCE_V2_CQ_DB); + roce_set_field(cq_db.parameter, V2_CQ_DB_CONS_IDX_M, + V2_CQ_DB_CONS_IDX_S, hr_cq->cons_index); + roce_set_field(cq_db.parameter, V2_CQ_DB_CMD_SN_M, + V2_CQ_DB_CMD_SN_S, 1); + + hns_roce_write64(hr_dev, (__le32 *)&cq_db, hr_cq->db_reg); + } } static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, struct hns_roce_srq *srq) { + struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device); struct hns_roce_v2_cqe *cqe, *dest; u32 prod_index; int nfreed = 0; @@ -3129,7 +3332,7 @@ static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, if (nfreed) { hr_cq->cons_index += nfreed; - hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index); + update_cq_db(hr_dev, hr_cq); } } @@ -3224,37 +3427,33 @@ static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq, { struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device); struct hns_roce_cq *hr_cq = to_hr_cq(ibcq); - u32 notification_flag; - __le32 doorbell[2]; + struct hns_roce_v2_db cq_db = {}; + u32 notify_flag; - doorbell[0] = 0; - doorbell[1] = 0; - - notification_flag = (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ? - V2_CQ_DB_REQ_NOT : V2_CQ_DB_REQ_NOT_SOL; /* - * flags = 0; Notification Flag = 1, next - * flags = 1; Notification Flag = 0, solocited + * flags = 0, then notify_flag : next + * flags = 1, then notify flag : solocited */ - roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_TAG_M, V2_DB_BYTE_4_TAG_S, - hr_cq->cqn); - roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_CMD_M, V2_DB_BYTE_4_CMD_S, - HNS_ROCE_V2_CQ_DB_NTR); - roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CONS_IDX_M, - V2_CQ_DB_PARAMETER_CONS_IDX_S, hr_cq->cons_index); - roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CMD_SN_M, - V2_CQ_DB_PARAMETER_CMD_SN_S, hr_cq->arm_sn & 0x3); - roce_set_bit(doorbell[1], V2_CQ_DB_PARAMETER_NOTIFY_S, - notification_flag); - - hns_roce_write64(hr_dev, doorbell, hr_cq->cq_db_l); + notify_flag = (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ? + V2_CQ_DB_REQ_NOT : V2_CQ_DB_REQ_NOT_SOL; + + roce_set_field(cq_db.byte_4, V2_DB_TAG_M, V2_DB_TAG_S, hr_cq->cqn); + roce_set_field(cq_db.byte_4, V2_DB_CMD_M, V2_DB_CMD_S, + HNS_ROCE_V2_CQ_DB_NOTIFY); + roce_set_field(cq_db.parameter, V2_CQ_DB_CONS_IDX_M, + V2_CQ_DB_CONS_IDX_S, hr_cq->cons_index); + roce_set_field(cq_db.parameter, V2_CQ_DB_CMD_SN_M, + V2_CQ_DB_CMD_SN_S, hr_cq->arm_sn); + roce_set_bit(cq_db.parameter, V2_CQ_DB_NOTIFY_TYPE_S, notify_flag); + + hns_roce_write64(hr_dev, (__le32 *)&cq_db, hr_cq->db_reg); return 0; } static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe, - struct hns_roce_qp **cur_qp, - struct ib_wc *wc) + struct hns_roce_qp *qp, + struct ib_wc *wc) { struct hns_roce_rinl_sge *sge_list; u32 wr_num, wr_cnt, sge_num; @@ -3263,11 +3462,11 @@ static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe, wr_num = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_WQE_INDX_M, V2_CQE_BYTE_4_WQE_INDX_S) & 0xffff; - wr_cnt = wr_num & ((*cur_qp)->rq.wqe_cnt - 1); + wr_cnt = wr_num & (qp->rq.wqe_cnt - 1); - sge_list = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sg_list; - sge_num = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sge_cnt; - wqe_buf = hns_roce_get_recv_wqe(*cur_qp, wr_cnt); + sge_list = qp->rq_inl_buf.wqe_list[wr_cnt].sg_list; + sge_num = qp->rq_inl_buf.wqe_list[wr_cnt].sge_cnt; + wqe_buf = hns_roce_get_recv_wqe(qp, wr_cnt); data_len = wc->byte_len; for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); sge_cnt++) { @@ -3401,21 +3600,205 @@ static void get_cqe_status(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp, init_flush_work(hr_dev, qp); } +static int get_cur_qp(struct hns_roce_cq *hr_cq, struct hns_roce_v2_cqe *cqe, + struct hns_roce_qp **cur_qp) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device); + struct hns_roce_qp *hr_qp = *cur_qp; + u32 qpn; + + qpn = roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M, + V2_CQE_BYTE_16_LCL_QPN_S) & + HNS_ROCE_V2_CQE_QPN_MASK; + + if (!hr_qp || qpn != hr_qp->qpn) { + hr_qp = __hns_roce_qp_lookup(hr_dev, qpn); + if (unlikely(!hr_qp)) { + ibdev_err(&hr_dev->ib_dev, + "CQ %06lx with entry for unknown QPN %06x\n", + hr_cq->cqn, qpn); + return -EINVAL; + } + *cur_qp = hr_qp; + } + + return 0; +} + +/* + * mapped-value = 1 + real-value + * The ib wc opcode's real value is start from 0, In order to distinguish + * between initialized and uninitialized map values, we plus 1 to the actual + * value when defining the mapping, so that the validity can be identified by + * checking whether the mapped value is greater than 0. + */ +#define HR_WC_OP_MAP(hr_key, ib_key) \ + [HNS_ROCE_V2_WQE_OP_ ## hr_key] = 1 + IB_WC_ ## ib_key + +static const u32 wc_send_op_map[] = { + HR_WC_OP_MAP(SEND, SEND), + HR_WC_OP_MAP(SEND_WITH_INV, SEND), + HR_WC_OP_MAP(SEND_WITH_IMM, SEND), + HR_WC_OP_MAP(RDMA_READ, RDMA_READ), + HR_WC_OP_MAP(RDMA_WRITE, RDMA_WRITE), + HR_WC_OP_MAP(RDMA_WRITE_WITH_IMM, RDMA_WRITE), + HR_WC_OP_MAP(LOCAL_INV, LOCAL_INV), + HR_WC_OP_MAP(ATOM_CMP_AND_SWAP, COMP_SWAP), + HR_WC_OP_MAP(ATOM_FETCH_AND_ADD, FETCH_ADD), + HR_WC_OP_MAP(ATOM_MSK_CMP_AND_SWAP, MASKED_COMP_SWAP), + HR_WC_OP_MAP(ATOM_MSK_FETCH_AND_ADD, MASKED_FETCH_ADD), + HR_WC_OP_MAP(FAST_REG_PMR, REG_MR), + HR_WC_OP_MAP(BIND_MW, REG_MR), +}; + +static int to_ib_wc_send_op(u32 hr_opcode) +{ + if (hr_opcode >= ARRAY_SIZE(wc_send_op_map)) + return -EINVAL; + + return wc_send_op_map[hr_opcode] ? wc_send_op_map[hr_opcode] - 1 : + -EINVAL; +} + +static const u32 wc_recv_op_map[] = { + HR_WC_OP_MAP(RDMA_WRITE_WITH_IMM, WITH_IMM), + HR_WC_OP_MAP(SEND, RECV), + HR_WC_OP_MAP(SEND_WITH_IMM, WITH_IMM), + HR_WC_OP_MAP(SEND_WITH_INV, RECV), +}; + +static int to_ib_wc_recv_op(u32 hr_opcode) +{ + if (hr_opcode >= ARRAY_SIZE(wc_recv_op_map)) + return -EINVAL; + + return wc_recv_op_map[hr_opcode] ? wc_recv_op_map[hr_opcode] - 1 : + -EINVAL; +} + +static void fill_send_wc(struct ib_wc *wc, struct hns_roce_v2_cqe *cqe) +{ + u32 hr_opcode; + int ib_opcode; + + wc->wc_flags = 0; + + hr_opcode = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M, + V2_CQE_BYTE_4_OPCODE_S) & 0x1f; + switch (hr_opcode) { + case HNS_ROCE_V2_WQE_OP_RDMA_READ: + wc->byte_len = le32_to_cpu(cqe->byte_cnt); + break; + case HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM: + case HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM: + wc->wc_flags |= IB_WC_WITH_IMM; + break; + case HNS_ROCE_V2_WQE_OP_LOCAL_INV: + wc->wc_flags |= IB_WC_WITH_INVALIDATE; + break; + case HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP: + case HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD: + case HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP: + case HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD: + wc->byte_len = 8; + break; + default: + break; + } + + ib_opcode = to_ib_wc_send_op(hr_opcode); + if (ib_opcode < 0) + wc->status = IB_WC_GENERAL_ERR; + else + wc->opcode = ib_opcode; +} + +static inline bool is_rq_inl_enabled(struct ib_wc *wc, u32 hr_opcode, + struct hns_roce_v2_cqe *cqe) +{ + return wc->qp->qp_type != IB_QPT_UD && + wc->qp->qp_type != IB_QPT_GSI && + (hr_opcode == HNS_ROCE_V2_OPCODE_SEND || + hr_opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_IMM || + hr_opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_INV) && + roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_RQ_INLINE_S); +} + +static int fill_recv_wc(struct ib_wc *wc, struct hns_roce_v2_cqe *cqe) +{ + struct hns_roce_qp *qp = to_hr_qp(wc->qp); + u32 hr_opcode; + int ib_opcode; + int ret; + + wc->byte_len = le32_to_cpu(cqe->byte_cnt); + + hr_opcode = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M, + V2_CQE_BYTE_4_OPCODE_S) & 0x1f; + switch (hr_opcode) { + case HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM: + case HNS_ROCE_V2_OPCODE_SEND_WITH_IMM: + wc->wc_flags = IB_WC_WITH_IMM; + wc->ex.imm_data = cpu_to_be32(le32_to_cpu(cqe->immtdata)); + break; + case HNS_ROCE_V2_OPCODE_SEND_WITH_INV: + wc->wc_flags = IB_WC_WITH_INVALIDATE; + wc->ex.invalidate_rkey = le32_to_cpu(cqe->rkey); + break; + default: + wc->wc_flags = 0; + } + + ib_opcode = to_ib_wc_recv_op(hr_opcode); + if (ib_opcode < 0) + wc->status = IB_WC_GENERAL_ERR; + else + wc->opcode = ib_opcode; + + if (is_rq_inl_enabled(wc, hr_opcode, cqe)) { + ret = hns_roce_handle_recv_inl_wqe(cqe, qp, wc); + if (unlikely(ret)) + return ret; + } + + wc->sl = roce_get_field(cqe->byte_32, V2_CQE_BYTE_32_SL_M, + V2_CQE_BYTE_32_SL_S); + wc->src_qp = roce_get_field(cqe->byte_32, V2_CQE_BYTE_32_RMT_QPN_M, + V2_CQE_BYTE_32_RMT_QPN_S); + wc->slid = 0; + wc->wc_flags |= roce_get_bit(cqe->byte_32, V2_CQE_BYTE_32_GRH_S) ? + IB_WC_GRH : 0; + wc->port_num = roce_get_field(cqe->byte_32, V2_CQE_BYTE_32_PORTN_M, + V2_CQE_BYTE_32_PORTN_S); + wc->pkey_index = 0; + + if (roce_get_bit(cqe->byte_28, V2_CQE_BYTE_28_VID_VLD_S)) { + wc->vlan_id = roce_get_field(cqe->byte_28, V2_CQE_BYTE_28_VID_M, + V2_CQE_BYTE_28_VID_S); + wc->wc_flags |= IB_WC_WITH_VLAN; + } else { + wc->vlan_id = 0xffff; + } + + wc->network_hdr_type = roce_get_field(cqe->byte_28, + V2_CQE_BYTE_28_PORT_TYPE_M, + V2_CQE_BYTE_28_PORT_TYPE_S); + + return 0; +} + static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq, struct hns_roce_qp **cur_qp, struct ib_wc *wc) { struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device); + struct hns_roce_qp *qp = *cur_qp; struct hns_roce_srq *srq = NULL; struct hns_roce_v2_cqe *cqe; - struct hns_roce_qp *hr_qp; struct hns_roce_wq *wq; int is_send; - u16 wqe_ctr; - u32 opcode; - u32 qpn; + u16 wqe_idx; int ret; - /* Find cqe according to consumer index */ cqe = get_sw_cqe_v2(hr_cq, hr_cq->cons_index); if (!cqe) return -EAGAIN; @@ -3424,189 +3807,50 @@ static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq, /* Memory barrier */ rmb(); - /* 0->SQ, 1->RQ */ - is_send = !roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S); - - qpn = roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M, - V2_CQE_BYTE_16_LCL_QPN_S); - - if (!*cur_qp || (qpn & HNS_ROCE_V2_CQE_QPN_MASK) != (*cur_qp)->qpn) { - hr_qp = __hns_roce_qp_lookup(hr_dev, qpn); - if (unlikely(!hr_qp)) { - ibdev_err(&hr_dev->ib_dev, - "CQ %06lx with entry for unknown QPN %06x\n", - hr_cq->cqn, qpn & HNS_ROCE_V2_CQE_QPN_MASK); - return -EINVAL; - } - *cur_qp = hr_qp; - } + ret = get_cur_qp(hr_cq, cqe, &qp); + if (ret) + return ret; - wc->qp = &(*cur_qp)->ibqp; + wc->qp = &qp->ibqp; wc->vendor_err = 0; + wqe_idx = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_WQE_INDX_M, + V2_CQE_BYTE_4_WQE_INDX_S); + + is_send = !roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S); if (is_send) { - wq = &(*cur_qp)->sq; - if ((*cur_qp)->sq_signal_bits) { - /* - * If sg_signal_bit is 1, - * firstly tail pointer updated to wqe - * which current cqe correspond to - */ - wqe_ctr = (u16)roce_get_field(cqe->byte_4, - V2_CQE_BYTE_4_WQE_INDX_M, - V2_CQE_BYTE_4_WQE_INDX_S); - wq->tail += (wqe_ctr - (u16)wq->tail) & + wq = &qp->sq; + + /* If sg_signal_bit is set, tail pointer will be updated to + * the WQE corresponding to the current CQE. + */ + if (qp->sq_signal_bits) + wq->tail += (wqe_idx - (u16)wq->tail) & (wq->wqe_cnt - 1); - } wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; ++wq->tail; - } else if ((*cur_qp)->ibqp.srq) { - srq = to_hr_srq((*cur_qp)->ibqp.srq); - wqe_ctr = (u16)roce_get_field(cqe->byte_4, - V2_CQE_BYTE_4_WQE_INDX_M, - V2_CQE_BYTE_4_WQE_INDX_S); - wc->wr_id = srq->wrid[wqe_ctr]; - hns_roce_free_srq_wqe(srq, wqe_ctr); - } else { - /* Update tail pointer, record wr_id */ - wq = &(*cur_qp)->rq; - wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; - ++wq->tail; - } - - get_cqe_status(hr_dev, *cur_qp, hr_cq, cqe, wc); - if (unlikely(wc->status != IB_WC_SUCCESS)) - return 0; - if (is_send) { - wc->wc_flags = 0; - /* SQ corresponding to CQE */ - switch (roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M, - V2_CQE_BYTE_4_OPCODE_S) & 0x1f) { - case HNS_ROCE_V2_WQE_OP_SEND: - wc->opcode = IB_WC_SEND; - break; - case HNS_ROCE_V2_WQE_OP_SEND_WITH_INV: - wc->opcode = IB_WC_SEND; - break; - case HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM: - wc->opcode = IB_WC_SEND; - wc->wc_flags |= IB_WC_WITH_IMM; - break; - case HNS_ROCE_V2_WQE_OP_RDMA_READ: - wc->opcode = IB_WC_RDMA_READ; - wc->byte_len = le32_to_cpu(cqe->byte_cnt); - break; - case HNS_ROCE_V2_WQE_OP_RDMA_WRITE: - wc->opcode = IB_WC_RDMA_WRITE; - break; - case HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM: - wc->opcode = IB_WC_RDMA_WRITE; - wc->wc_flags |= IB_WC_WITH_IMM; - break; - case HNS_ROCE_V2_WQE_OP_LOCAL_INV: - wc->opcode = IB_WC_LOCAL_INV; - wc->wc_flags |= IB_WC_WITH_INVALIDATE; - break; - case HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP: - wc->opcode = IB_WC_COMP_SWAP; - wc->byte_len = 8; - break; - case HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD: - wc->opcode = IB_WC_FETCH_ADD; - wc->byte_len = 8; - break; - case HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP: - wc->opcode = IB_WC_MASKED_COMP_SWAP; - wc->byte_len = 8; - break; - case HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD: - wc->opcode = IB_WC_MASKED_FETCH_ADD; - wc->byte_len = 8; - break; - case HNS_ROCE_V2_WQE_OP_FAST_REG_PMR: - wc->opcode = IB_WC_REG_MR; - break; - case HNS_ROCE_V2_WQE_OP_BIND_MW: - wc->opcode = IB_WC_REG_MR; - break; - default: - wc->status = IB_WC_GENERAL_ERR; - break; - } + fill_send_wc(wc, cqe); } else { - /* RQ correspond to CQE */ - wc->byte_len = le32_to_cpu(cqe->byte_cnt); - - opcode = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M, - V2_CQE_BYTE_4_OPCODE_S); - switch (opcode & 0x1f) { - case HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM: - wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; - wc->wc_flags = IB_WC_WITH_IMM; - wc->ex.imm_data = - cpu_to_be32(le32_to_cpu(cqe->immtdata)); - break; - case HNS_ROCE_V2_OPCODE_SEND: - wc->opcode = IB_WC_RECV; - wc->wc_flags = 0; - break; - case HNS_ROCE_V2_OPCODE_SEND_WITH_IMM: - wc->opcode = IB_WC_RECV; - wc->wc_flags = IB_WC_WITH_IMM; - wc->ex.imm_data = - cpu_to_be32(le32_to_cpu(cqe->immtdata)); - break; - case HNS_ROCE_V2_OPCODE_SEND_WITH_INV: - wc->opcode = IB_WC_RECV; - wc->wc_flags = IB_WC_WITH_INVALIDATE; - wc->ex.invalidate_rkey = le32_to_cpu(cqe->rkey); - break; - default: - wc->status = IB_WC_GENERAL_ERR; - break; - } - - if ((wc->qp->qp_type == IB_QPT_RC || - wc->qp->qp_type == IB_QPT_UC) && - (opcode == HNS_ROCE_V2_OPCODE_SEND || - opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_IMM || - opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_INV) && - (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_RQ_INLINE_S))) { - ret = hns_roce_handle_recv_inl_wqe(cqe, cur_qp, wc); - if (unlikely(ret)) - return -EAGAIN; - } - - wc->sl = (u8)roce_get_field(cqe->byte_32, V2_CQE_BYTE_32_SL_M, - V2_CQE_BYTE_32_SL_S); - wc->src_qp = (u8)roce_get_field(cqe->byte_32, - V2_CQE_BYTE_32_RMT_QPN_M, - V2_CQE_BYTE_32_RMT_QPN_S); - wc->slid = 0; - wc->wc_flags |= (roce_get_bit(cqe->byte_32, - V2_CQE_BYTE_32_GRH_S) ? - IB_WC_GRH : 0); - wc->port_num = roce_get_field(cqe->byte_32, - V2_CQE_BYTE_32_PORTN_M, V2_CQE_BYTE_32_PORTN_S); - wc->pkey_index = 0; - - if (roce_get_bit(cqe->byte_28, V2_CQE_BYTE_28_VID_VLD_S)) { - wc->vlan_id = (u16)roce_get_field(cqe->byte_28, - V2_CQE_BYTE_28_VID_M, - V2_CQE_BYTE_28_VID_S); - wc->wc_flags |= IB_WC_WITH_VLAN; + if (qp->ibqp.srq) { + srq = to_hr_srq(qp->ibqp.srq); + wc->wr_id = srq->wrid[wqe_idx]; + hns_roce_free_srq_wqe(srq, wqe_idx); } else { - wc->vlan_id = 0xffff; + wq = &qp->rq; + wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; + ++wq->tail; } - wc->network_hdr_type = roce_get_field(cqe->byte_28, - V2_CQE_BYTE_28_PORT_TYPE_M, - V2_CQE_BYTE_28_PORT_TYPE_S); + ret = fill_recv_wc(wc, cqe); } - return 0; + get_cqe_status(hr_dev, qp, hr_cq, cqe, wc); + if (unlikely(wc->status != IB_WC_SUCCESS)) + return 0; + + return ret; } static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries, @@ -3638,7 +3882,7 @@ static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries, } if (npolled) - hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index); + update_cq_db(hr_dev, hr_cq); out: spin_unlock_irqrestore(&hr_cq->lock, flags); @@ -3647,12 +3891,9 @@ out: } static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type, - int step_idx) + int step_idx, u16 *mbox_op) { - int op; - - if (type == HEM_TYPE_SCCC && step_idx) - return -EINVAL; + u16 op; switch (type) { case HEM_TYPE_QPC: @@ -3677,51 +3918,49 @@ static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type, op = HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0; break; default: - dev_warn(hr_dev->dev, - "table %u not to be written by mailbox!\n", type); + dev_warn(hr_dev->dev, "failed to check hem type %u.\n", type); return -EINVAL; } - return op + step_idx; + *mbox_op = op + step_idx; + + return 0; } -static int set_hem_to_hw(struct hns_roce_dev *hr_dev, int obj, u64 bt_ba, - u32 hem_type, int step_idx) +static int config_gmv_ba_to_hw(struct hns_roce_dev *hr_dev, unsigned long obj, + dma_addr_t base_addr) { - struct hns_roce_cmd_mailbox *mailbox; struct hns_roce_cmq_desc desc; - struct hns_roce_cfg_gmv_bt *gmv_bt = - (struct hns_roce_cfg_gmv_bt *)desc.data; - int ret; - int op; + struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data; + u32 idx = obj / (HNS_HW_PAGE_SIZE / hr_dev->caps.gmv_entry_sz); + u64 addr = to_hr_hw_page_addr(base_addr); - if (hem_type == HEM_TYPE_GMV) { - hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT, - false); + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT, false); - gmv_bt->gmv_ba_l = cpu_to_le32(bt_ba >> HNS_HW_PAGE_SHIFT); - gmv_bt->gmv_ba_h = cpu_to_le32(bt_ba >> (HNS_HW_PAGE_SHIFT + - 32)); - gmv_bt->gmv_bt_idx = cpu_to_le32(obj / - (HNS_HW_PAGE_SIZE / hr_dev->caps.gmv_entry_sz)); + hr_reg_write(req, CFG_GMV_BT_BA_L, lower_32_bits(addr)); + hr_reg_write(req, CFG_GMV_BT_BA_H, upper_32_bits(addr)); + hr_reg_write(req, CFG_GMV_BT_IDX, idx); - return hns_roce_cmq_send(hr_dev, &desc, 1); - } + return hns_roce_cmq_send(hr_dev, &desc, 1); +} - op = get_op_for_set_hem(hr_dev, hem_type, step_idx); - if (op < 0) - return 0; +static int set_hem_to_hw(struct hns_roce_dev *hr_dev, int obj, + dma_addr_t base_addr, u32 hem_type, int step_idx) +{ + int ret; + u16 op; - mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); + if (unlikely(hem_type == HEM_TYPE_GMV)) + return config_gmv_ba_to_hw(hr_dev, obj, base_addr); - ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, obj, - 0, op, HNS_ROCE_CMD_TIMEOUT_MSECS); + if (unlikely(hem_type == HEM_TYPE_SCCC && step_idx)) + return 0; - hns_roce_free_cmd_mailbox(hr_dev, mailbox); + ret = get_op_for_set_hem(hr_dev, hem_type, step_idx, &op); + if (ret < 0) + return ret; - return ret; + return config_hem_ba_to_hw(hr_dev, obj, base_addr, op); } static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev, @@ -3911,6 +4150,16 @@ static void set_qpc_wqe_cnt(struct hns_roce_qp *hr_qp, ilog2(hr_qp->rq.wqe_cnt)); } +static inline int get_cqn(struct ib_cq *ib_cq) +{ + return ib_cq ? to_hr_cq(ib_cq)->cqn : 0; +} + +static inline int get_pdn(struct ib_pd *ib_pd) +{ + return ib_pd ? to_hr_pd(ib_pd)->pdn : 0; +} + static void modify_qp_reset_to_init(struct ib_qp *ibqp, const struct ib_qp_attr *attr, int attr_mask, @@ -3927,13 +4176,13 @@ static void modify_qp_reset_to_init(struct ib_qp *ibqp, * 0 at the same time, else set them to 0x1. */ roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M, - V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type)); + V2_QPC_BYTE_4_TST_S, to_hr_qp_type(ibqp->qp_type)); roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M, V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn); roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M, - V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn); + V2_QPC_BYTE_16_PD_S, get_pdn(ibqp->pd)); roce_set_field(context->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M, V2_QPC_BYTE_20_RQWS_S, ilog2(hr_qp->rq.max_gs)); @@ -3944,6 +4193,13 @@ static void modify_qp_reset_to_init(struct ib_qp *ibqp, roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M, V2_QPC_BYTE_24_VLAN_ID_S, 0xfff); + if (ibqp->qp_type == IB_QPT_XRC_TGT) { + context->qkey_xrcd = cpu_to_le32(hr_qp->xrcdn); + + roce_set_bit(context->byte_80_rnr_rx_cqn, + V2_QPC_BYTE_80_XRC_QP_TYPE_S, 1); + } + if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) roce_set_bit(context->byte_68_rq_db, V2_QPC_BYTE_68_RQ_RECORD_EN_S, 1); @@ -3954,23 +4210,27 @@ static void modify_qp_reset_to_init(struct ib_qp *ibqp, ((u32)hr_qp->rdb.dma) >> 1); context->rq_db_record_addr = cpu_to_le32(hr_qp->rdb.dma >> 32); - roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S, - (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) ? 1 : 0); + if (ibqp->qp_type != IB_QPT_UD && ibqp->qp_type != IB_QPT_GSI) + roce_set_bit(context->byte_76_srqn_op_en, + V2_QPC_BYTE_76_RQIE_S, + !!(hr_dev->caps.flags & + HNS_ROCE_CAP_FLAG_RQ_INLINE)); roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M, - V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn); + V2_QPC_BYTE_80_RX_CQN_S, get_cqn(ibqp->recv_cq)); + if (ibqp->srq) { + roce_set_bit(context->byte_76_srqn_op_en, + V2_QPC_BYTE_76_SRQ_EN_S, 1); roce_set_field(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, to_hr_srq(ibqp->srq)->srqn); - roce_set_bit(context->byte_76_srqn_op_en, - V2_QPC_BYTE_76_SRQ_EN_S, 1); } roce_set_bit(context->byte_172_sq_psn, V2_QPC_BYTE_172_FRE_S, 1); roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M, - V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn); + V2_QPC_BYTE_252_TX_CQN_S, get_cqn(ibqp->send_cq)); if (hr_dev->caps.qpc_sz < HNS_ROCE_V3_QPC_SZ) return; @@ -3993,22 +4253,23 @@ static void modify_qp_init_to_init(struct ib_qp *ibqp, * 0 at the same time, else set them to 0x1. */ roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M, - V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type)); + V2_QPC_BYTE_4_TST_S, to_hr_qp_type(ibqp->qp_type)); roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M, V2_QPC_BYTE_4_TST_S, 0); roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M, - V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn); + V2_QPC_BYTE_16_PD_S, get_pdn(ibqp->pd)); + roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M, V2_QPC_BYTE_16_PD_S, 0); roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M, - V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn); + V2_QPC_BYTE_80_RX_CQN_S, get_cqn(ibqp->recv_cq)); roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M, V2_QPC_BYTE_80_RX_CQN_S, 0); roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M, - V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn); + V2_QPC_BYTE_252_TX_CQN_S, get_cqn(ibqp->send_cq)); roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M, V2_QPC_BYTE_252_TX_CQN_S, 0); @@ -4133,17 +4394,6 @@ static int config_qp_rq_buf(struct hns_roce_dev *hr_dev, V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M, V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, 0); - roce_set_field(context->byte_84_rq_ci_pi, - V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M, - V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, hr_qp->rq.head); - roce_set_field(qpc_mask->byte_84_rq_ci_pi, - V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M, - V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0); - - roce_set_field(qpc_mask->byte_84_rq_ci_pi, - V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M, - V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0); - return 0; } @@ -4240,7 +4490,7 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, u64 *mtts; u8 *dmac; u8 *smac; - int port; + u32 port; int ret; ret = config_qp_rq_buf(hr_dev, hr_qp, context, qpc_mask); @@ -4454,6 +4704,143 @@ static inline u16 get_udp_sport(u32 fl, u32 lqpn, u32 rqpn) return rdma_flow_label_to_udp_sport(fl); } +static int get_dip_ctx_idx(struct ib_qp *ibqp, const struct ib_qp_attr *attr, + u32 *dip_idx) +{ + const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_dip *hr_dip; + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(&hr_dev->dip_list_lock, flags); + + list_for_each_entry(hr_dip, &hr_dev->dip_list, node) { + if (!memcmp(grh->dgid.raw, hr_dip->dgid, 16)) + goto out; + } + + /* If no dgid is found, a new dip and a mapping between dgid and + * dip_idx will be created. + */ + hr_dip = kzalloc(sizeof(*hr_dip), GFP_ATOMIC); + if (!hr_dip) { + ret = -ENOMEM; + goto out; + } + + memcpy(hr_dip->dgid, grh->dgid.raw, sizeof(grh->dgid.raw)); + hr_dip->dip_idx = *dip_idx = ibqp->qp_num; + list_add_tail(&hr_dip->node, &hr_dev->dip_list); + +out: + spin_unlock_irqrestore(&hr_dev->dip_list_lock, flags); + return ret; +} + +enum { + CONG_DCQCN, + CONG_WINDOW, +}; + +enum { + UNSUPPORT_CONG_LEVEL, + SUPPORT_CONG_LEVEL, +}; + +enum { + CONG_LDCP, + CONG_HC3, +}; + +enum { + DIP_INVALID, + DIP_VALID, +}; + +static int check_cong_type(struct ib_qp *ibqp, + struct hns_roce_congestion_algorithm *cong_alg) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + + /* different congestion types match different configurations */ + switch (hr_dev->caps.cong_type) { + case CONG_TYPE_DCQCN: + cong_alg->alg_sel = CONG_DCQCN; + cong_alg->alg_sub_sel = UNSUPPORT_CONG_LEVEL; + cong_alg->dip_vld = DIP_INVALID; + break; + case CONG_TYPE_LDCP: + cong_alg->alg_sel = CONG_WINDOW; + cong_alg->alg_sub_sel = CONG_LDCP; + cong_alg->dip_vld = DIP_INVALID; + break; + case CONG_TYPE_HC3: + cong_alg->alg_sel = CONG_WINDOW; + cong_alg->alg_sub_sel = CONG_HC3; + cong_alg->dip_vld = DIP_INVALID; + break; + case CONG_TYPE_DIP: + cong_alg->alg_sel = CONG_DCQCN; + cong_alg->alg_sub_sel = UNSUPPORT_CONG_LEVEL; + cong_alg->dip_vld = DIP_VALID; + break; + default: + ibdev_err(&hr_dev->ib_dev, + "error type(%u) for congestion selection.\n", + hr_dev->caps.cong_type); + return -EINVAL; + } + + return 0; +} + +static int fill_cong_field(struct ib_qp *ibqp, const struct ib_qp_attr *attr, + struct hns_roce_v2_qp_context *context, + struct hns_roce_v2_qp_context *qpc_mask) +{ + const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); + struct hns_roce_congestion_algorithm cong_field; + struct ib_device *ibdev = ibqp->device; + struct hns_roce_dev *hr_dev = to_hr_dev(ibdev); + u32 dip_idx = 0; + int ret; + + if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 || + grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE) + return 0; + + ret = check_cong_type(ibqp, &cong_field); + if (ret) + return ret; + + hr_reg_write(context, QPC_CONG_ALGO_TMPL_ID, hr_dev->cong_algo_tmpl_id + + hr_dev->caps.cong_type * HNS_ROCE_CONG_SIZE); + hr_reg_write(qpc_mask, QPC_CONG_ALGO_TMPL_ID, 0); + hr_reg_write(&context->ext, QPCEX_CONG_ALG_SEL, cong_field.alg_sel); + hr_reg_write(&qpc_mask->ext, QPCEX_CONG_ALG_SEL, 0); + hr_reg_write(&context->ext, QPCEX_CONG_ALG_SUB_SEL, + cong_field.alg_sub_sel); + hr_reg_write(&qpc_mask->ext, QPCEX_CONG_ALG_SUB_SEL, 0); + hr_reg_write(&context->ext, QPCEX_DIP_CTX_IDX_VLD, cong_field.dip_vld); + hr_reg_write(&qpc_mask->ext, QPCEX_DIP_CTX_IDX_VLD, 0); + + /* if dip is disabled, there is no need to set dip idx */ + if (cong_field.dip_vld == 0) + return 0; + + ret = get_dip_ctx_idx(ibqp, attr, &dip_idx); + if (ret) { + ibdev_err(ibdev, "failed to fill cong field, ret = %d.\n", ret); + return ret; + } + + hr_reg_write(&context->ext, QPCEX_DIP_CTX_IDX, dip_idx); + hr_reg_write(&qpc_mask->ext, QPCEX_DIP_CTX_IDX, 0); + + return 0; +} + static int hns_roce_v2_set_path(struct ib_qp *ibqp, const struct ib_qp_attr *attr, int attr_mask, @@ -4537,6 +4924,10 @@ static int hns_roce_v2_set_path(struct ib_qp *ibqp, roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M, V2_QPC_BYTE_24_HOP_LIMIT_S, 0); + ret = fill_cong_field(ibqp, attr, context, qpc_mask); + if (ret) + return ret; + roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M, V2_QPC_BYTE_24_TC_S, get_tclass(&attr->ah_attr.grh)); roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M, @@ -4687,7 +5078,6 @@ static int hns_roce_v2_set_opt_fields(struct ib_qp *ibqp, V2_QPC_BYTE_244_RNR_CNT_S, 0); } - /* RC&UC&UD required attr */ if (attr_mask & IB_QP_SQ_PSN) { roce_set_field(context->byte_172_sq_psn, V2_QPC_BYTE_172_SQ_CUR_PSN_M, @@ -4765,7 +5155,6 @@ static int hns_roce_v2_set_opt_fields(struct ib_qp *ibqp, V2_QPC_BYTE_80_MIN_RNR_TIME_S, 0); } - /* RC&UC required attr */ if (attr_mask & IB_QP_RQ_PSN) { roce_set_field(context->byte_108_rx_reqepsn, V2_QPC_BYTE_108_RX_REQ_EPSN_M, @@ -4808,6 +5197,29 @@ static void hns_roce_v2_record_opt_fields(struct ib_qp *ibqp, } } +static void clear_qp(struct hns_roce_qp *hr_qp) +{ + struct ib_qp *ibqp = &hr_qp->ibqp; + + if (ibqp->send_cq) + hns_roce_v2_cq_clean(to_hr_cq(ibqp->send_cq), + hr_qp->qpn, NULL); + + if (ibqp->recv_cq && ibqp->recv_cq != ibqp->send_cq) + hns_roce_v2_cq_clean(to_hr_cq(ibqp->recv_cq), + hr_qp->qpn, ibqp->srq ? + to_hr_srq(ibqp->srq) : NULL); + + if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) + *hr_qp->rdb.db_record = 0; + + hr_qp->rq.head = 0; + hr_qp->rq.tail = 0; + hr_qp->sq.head = 0; + hr_qp->sq.tail = 0; + hr_qp->next_sge = 0; +} + static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, int attr_mask, enum ib_qp_state cur_state, @@ -4842,19 +5254,23 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, /* When QP state is err, SQ and RQ WQE should be flushed */ if (new_state == IB_QPS_ERR) { - spin_lock_irqsave(&hr_qp->sq.lock, sq_flag); - hr_qp->state = IB_QPS_ERR; - roce_set_field(context->byte_160_sq_ci_pi, - V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M, - V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, - hr_qp->sq.head); - roce_set_field(qpc_mask->byte_160_sq_ci_pi, - V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M, - V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0); - spin_unlock_irqrestore(&hr_qp->sq.lock, sq_flag); - - if (!ibqp->srq) { + if (ibqp->qp_type != IB_QPT_XRC_TGT) { + spin_lock_irqsave(&hr_qp->sq.lock, sq_flag); + hr_qp->state = IB_QPS_ERR; + roce_set_field(context->byte_160_sq_ci_pi, + V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M, + V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, + hr_qp->sq.head); + roce_set_field(qpc_mask->byte_160_sq_ci_pi, + V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M, + V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0); + spin_unlock_irqrestore(&hr_qp->sq.lock, sq_flag); + } + + if (!ibqp->srq && ibqp->qp_type != IB_QPT_XRC_INI && + ibqp->qp_type != IB_QPT_XRC_TGT) { spin_lock_irqsave(&hr_qp->rq.lock, rq_flag); + hr_qp->state = IB_QPS_ERR; roce_set_field(context->byte_84_rq_ci_pi, V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M, V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, @@ -4873,7 +5289,8 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, goto out; roce_set_bit(context->byte_108_rx_reqepsn, V2_QPC_BYTE_108_INV_CREDIT_S, - ibqp->srq ? 1 : 0); + ((to_hr_qp_type(hr_qp->ibqp.qp_type) == SERV_TYPE_XRC) || + ibqp->srq) ? 1 : 0); roce_set_bit(qpc_mask->byte_108_rx_reqepsn, V2_QPC_BYTE_108_INV_CREDIT_S, 0); @@ -4894,21 +5311,8 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, hns_roce_v2_record_opt_fields(ibqp, attr, attr_mask); - if (new_state == IB_QPS_RESET && !ibqp->uobject) { - hns_roce_v2_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn, - ibqp->srq ? to_hr_srq(ibqp->srq) : NULL); - if (ibqp->send_cq != ibqp->recv_cq) - hns_roce_v2_cq_clean(to_hr_cq(ibqp->send_cq), - hr_qp->qpn, NULL); - - hr_qp->rq.head = 0; - hr_qp->rq.tail = 0; - hr_qp->sq.head = 0; - hr_qp->sq.tail = 0; - hr_qp->next_sge = 0; - if (hr_qp->rq.wqe_cnt) - *hr_qp->rdb.db_record = 0; - } + if (new_state == IB_QPS_RESET && !ibqp->uobject) + clear_qp(hr_qp); out: return ret; @@ -5019,7 +5423,8 @@ static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, V2_QPC_BYTE_76_ATE_S)) << V2_QP_ATE_S); if (hr_qp->ibqp.qp_type == IB_QPT_RC || - hr_qp->ibqp.qp_type == IB_QPT_UC) { + hr_qp->ibqp.qp_type == IB_QPT_XRC_INI || + hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT) { struct ib_global_route *grh = rdma_ah_retrieve_grh(&qp_attr->ah_attr); @@ -5051,6 +5456,7 @@ static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, qp_attr->max_dest_rd_atomic = 1 << roce_get_field(context.byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M, V2_QPC_BYTE_140_RR_MAX_S); + qp_attr->min_rnr_timer = (u8)roce_get_field(context.byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_MIN_RNR_TIME_M, V2_QPC_BYTE_80_MIN_RNR_TIME_S); @@ -5068,6 +5474,7 @@ done: qp_attr->cur_qp_state = qp_attr->qp_state; qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt; qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge; + qp_attr->cap.max_inline_data = hr_qp->max_inline_data; if (!ibqp->uobject) { qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt; @@ -5085,6 +5492,15 @@ out: return ret; } +static inline int modify_qp_is_ok(struct hns_roce_qp *hr_qp) +{ + return ((hr_qp->ibqp.qp_type == IB_QPT_RC || + hr_qp->ibqp.qp_type == IB_QPT_UD || + hr_qp->ibqp.qp_type == IB_QPT_XRC_INI || + hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT) && + hr_qp->state != IB_QPS_RESET); +} + static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, struct ib_udata *udata) @@ -5094,9 +5510,7 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, unsigned long flags; int ret = 0; - if ((hr_qp->ibqp.qp_type == IB_QPT_RC || - hr_qp->ibqp.qp_type == IB_QPT_UD) && - hr_qp->state != IB_QPS_RESET) { + if (modify_qp_is_ok(hr_qp)) { /* Modify qp to reset before destroying qp */ ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0, hr_qp->state, IB_QPS_RESET); @@ -5275,9 +5689,11 @@ static int hns_roce_v2_write_srqc(struct hns_roce_srq *srq, void *mb_buf) } hr_reg_write(ctx, SRQC_SRQ_ST, 1); + hr_reg_write(ctx, SRQC_SRQ_TYPE, + !!(srq->ibsrq.srq_type == IB_SRQT_XRC)); hr_reg_write(ctx, SRQC_PD, to_hr_pd(srq->ibsrq.pd)->pdn); hr_reg_write(ctx, SRQC_SRQN, srq->srqn); - hr_reg_write(ctx, SRQC_XRCD, 0); + hr_reg_write(ctx, SRQC_XRCD, srq->xrcdn); hr_reg_write(ctx, SRQC_XRC_CQN, srq->cqn); hr_reg_write(ctx, SRQC_SHIFT, ilog2(srq->wqe_cnt)); hr_reg_write(ctx, SRQC_RQWS, @@ -5481,6 +5897,12 @@ static void hns_roce_irq_work_handle(struct work_struct *work) case HNS_ROCE_EVENT_TYPE_FLR: ibdev_warn(ibdev, "Function level reset.\n"); break; + case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION: + ibdev_err(ibdev, "xrc domain violation error.\n"); + break; + case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH: + ibdev_err(ibdev, "invalid xrceth error.\n"); + break; default: break; } @@ -5505,33 +5927,30 @@ static void hns_roce_v2_init_irq_work(struct hns_roce_dev *hr_dev, queue_work(hr_dev->irq_workq, &(irq_work->work)); } -static void set_eq_cons_index_v2(struct hns_roce_eq *eq) +static void update_eq_db(struct hns_roce_eq *eq) { struct hns_roce_dev *hr_dev = eq->hr_dev; - __le32 doorbell[2] = {}; + struct hns_roce_v2_db eq_db = {}; if (eq->type_flag == HNS_ROCE_AEQ) { - roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M, - HNS_ROCE_V2_EQ_DB_CMD_S, + roce_set_field(eq_db.byte_4, V2_EQ_DB_CMD_M, V2_EQ_DB_CMD_S, eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ? HNS_ROCE_EQ_DB_CMD_AEQ : HNS_ROCE_EQ_DB_CMD_AEQ_ARMED); } else { - roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_TAG_M, - HNS_ROCE_V2_EQ_DB_TAG_S, eq->eqn); + roce_set_field(eq_db.byte_4, V2_EQ_DB_TAG_M, V2_EQ_DB_TAG_S, + eq->eqn); - roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M, - HNS_ROCE_V2_EQ_DB_CMD_S, + roce_set_field(eq_db.byte_4, V2_EQ_DB_CMD_M, V2_EQ_DB_CMD_S, eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ? HNS_ROCE_EQ_DB_CMD_CEQ : HNS_ROCE_EQ_DB_CMD_CEQ_ARMED); } - roce_set_field(doorbell[1], HNS_ROCE_V2_EQ_DB_PARA_M, - HNS_ROCE_V2_EQ_DB_PARA_S, - (eq->cons_index & HNS_ROCE_V2_CONS_IDX_M)); + roce_set_field(eq_db.parameter, V2_EQ_DB_CONS_IDX_M, + V2_EQ_DB_CONS_IDX_S, eq->cons_index); - hns_roce_write64(hr_dev, doorbell, eq->doorbell); + hns_roce_write64(hr_dev, (__le32 *)&eq_db, eq->db_reg); } static struct hns_roce_aeqe *next_aeqe_sw_v2(struct hns_roce_eq *eq) @@ -5581,6 +6000,8 @@ static int hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev, case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH: case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: + case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION: + case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH: hns_roce_qp_event(hr_dev, queue_num, event_type); break; case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH: @@ -5616,7 +6037,7 @@ static int hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev, aeqe = next_aeqe_sw_v2(eq); } - set_eq_cons_index_v2(eq); + update_eq_db(eq); return aeqe_found; } @@ -5656,7 +6077,7 @@ static int hns_roce_v2_ceq_int(struct hns_roce_dev *hr_dev, ceqe = next_ceqe_sw_v2(eq); } - set_eq_cons_index_v2(eq); + update_eq_db(eq); return ceqe_found; } @@ -5710,58 +6131,34 @@ static irqreturn_t hns_roce_v2_msix_interrupt_abn(int irq, void *dev_id) roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en); int_work = 1; - } else if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S)) { - dev_err(dev, "BUS ERR!\n"); - - int_st |= 1 << HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S; - roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st); - - int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S; - roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en); - - int_work = 1; - } else if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S)) { - dev_err(dev, "OTHER ERR!\n"); + } else if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_RAS_INT_S)) { + dev_err(dev, "RAS interrupt!\n"); - int_st |= 1 << HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S; + int_st |= 1 << HNS_ROCE_V2_VF_INT_ST_RAS_INT_S; roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st); int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S; roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en); int_work = 1; - } else + } else { dev_err(dev, "There is no abnormal irq found!\n"); + } return IRQ_RETVAL(int_work); } static void hns_roce_v2_int_mask_enable(struct hns_roce_dev *hr_dev, - int eq_num, int enable_flag) + int eq_num, u32 enable_flag) { int i; - if (enable_flag == EQ_ENABLE) { - for (i = 0; i < eq_num; i++) - roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG + - i * EQ_REG_OFFSET, - HNS_ROCE_V2_VF_EVENT_INT_EN_M); - - roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, - HNS_ROCE_V2_VF_ABN_INT_EN_M); - roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG, - HNS_ROCE_V2_VF_ABN_INT_CFG_M); - } else { - for (i = 0; i < eq_num; i++) - roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG + - i * EQ_REG_OFFSET, - HNS_ROCE_V2_VF_EVENT_INT_EN_M & 0x0); + for (i = 0; i < eq_num; i++) + roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG + + i * EQ_REG_OFFSET, enable_flag); - roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, - HNS_ROCE_V2_VF_ABN_INT_EN_M & 0x0); - roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG, - HNS_ROCE_V2_VF_ABN_INT_CFG_M & 0x0); - } + roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, enable_flag); + roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG, enable_flag); } static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev, int eqn) @@ -5786,6 +6183,16 @@ static void free_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq) hns_roce_mtr_destroy(hr_dev, &eq->mtr); } +static void init_eq_config(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq) +{ + eq->db_reg = hr_dev->reg_base + ROCEE_VF_EQ_DB_CFG0_REG; + eq->cons_index = 0; + eq->over_ignore = HNS_ROCE_V2_EQ_OVER_IGNORE_0; + eq->coalesce = HNS_ROCE_V2_EQ_COALESCE_0; + eq->arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED; + eq->shift = ilog2((unsigned int)eq->entries); +} + static int config_eqc(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq, void *mb_buf) { @@ -5797,13 +6204,7 @@ static int config_eqc(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq, eqc = mb_buf; memset(eqc, 0, sizeof(struct hns_roce_eq_context)); - /* init eqc */ - eq->doorbell = hr_dev->reg_base + ROCEE_VF_EQ_DB_CFG0_REG; - eq->cons_index = 0; - eq->over_ignore = HNS_ROCE_V2_EQ_OVER_IGNORE_0; - eq->coalesce = HNS_ROCE_V2_EQ_COALESCE_0; - eq->arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED; - eq->shift = ilog2((unsigned int)eq->entries); + init_eq_config(hr_dev, eq); /* if not multi-hop, eqe buffer only use one trunk */ count = hns_roce_mtr_find(hr_dev, &eq->mtr, 0, eqe_ba, MTT_MIN_COUNT, @@ -5813,102 +6214,34 @@ static int config_eqc(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq, return -ENOBUFS; } - /* set eqc state */ - roce_set_field(eqc->byte_4, HNS_ROCE_EQC_EQ_ST_M, HNS_ROCE_EQC_EQ_ST_S, - HNS_ROCE_V2_EQ_STATE_VALID); - - /* set eqe hop num */ - roce_set_field(eqc->byte_4, HNS_ROCE_EQC_HOP_NUM_M, - HNS_ROCE_EQC_HOP_NUM_S, eq->hop_num); - - /* set eqc over_ignore */ - roce_set_field(eqc->byte_4, HNS_ROCE_EQC_OVER_IGNORE_M, - HNS_ROCE_EQC_OVER_IGNORE_S, eq->over_ignore); - - /* set eqc coalesce */ - roce_set_field(eqc->byte_4, HNS_ROCE_EQC_COALESCE_M, - HNS_ROCE_EQC_COALESCE_S, eq->coalesce); - - /* set eqc arm_state */ - roce_set_field(eqc->byte_4, HNS_ROCE_EQC_ARM_ST_M, - HNS_ROCE_EQC_ARM_ST_S, eq->arm_st); - - /* set eqn */ - roce_set_field(eqc->byte_4, HNS_ROCE_EQC_EQN_M, HNS_ROCE_EQC_EQN_S, - eq->eqn); - - /* set eqe_cnt */ - roce_set_field(eqc->byte_4, HNS_ROCE_EQC_EQE_CNT_M, - HNS_ROCE_EQC_EQE_CNT_S, HNS_ROCE_EQ_INIT_EQE_CNT); - - /* set eqe_ba_pg_sz */ - roce_set_field(eqc->byte_8, HNS_ROCE_EQC_BA_PG_SZ_M, - HNS_ROCE_EQC_BA_PG_SZ_S, - to_hr_hw_page_shift(eq->mtr.hem_cfg.ba_pg_shift)); - - /* set eqe_buf_pg_sz */ - roce_set_field(eqc->byte_8, HNS_ROCE_EQC_BUF_PG_SZ_M, - HNS_ROCE_EQC_BUF_PG_SZ_S, - to_hr_hw_page_shift(eq->mtr.hem_cfg.buf_pg_shift)); - - /* set eq_producer_idx */ - roce_set_field(eqc->byte_8, HNS_ROCE_EQC_PROD_INDX_M, - HNS_ROCE_EQC_PROD_INDX_S, HNS_ROCE_EQ_INIT_PROD_IDX); - - /* set eq_max_cnt */ - roce_set_field(eqc->byte_12, HNS_ROCE_EQC_MAX_CNT_M, - HNS_ROCE_EQC_MAX_CNT_S, eq->eq_max_cnt); - - /* set eq_period */ - roce_set_field(eqc->byte_12, HNS_ROCE_EQC_PERIOD_M, - HNS_ROCE_EQC_PERIOD_S, eq->eq_period); - - /* set eqe_report_timer */ - roce_set_field(eqc->eqe_report_timer, HNS_ROCE_EQC_REPORT_TIMER_M, - HNS_ROCE_EQC_REPORT_TIMER_S, - HNS_ROCE_EQ_INIT_REPORT_TIMER); - - /* set bt_ba [34:3] */ - roce_set_field(eqc->eqe_ba0, HNS_ROCE_EQC_EQE_BA_L_M, - HNS_ROCE_EQC_EQE_BA_L_S, bt_ba >> 3); - - /* set bt_ba [64:35] */ - roce_set_field(eqc->eqe_ba1, HNS_ROCE_EQC_EQE_BA_H_M, - HNS_ROCE_EQC_EQE_BA_H_S, bt_ba >> 35); - - /* set eq shift */ - roce_set_field(eqc->byte_28, HNS_ROCE_EQC_SHIFT_M, HNS_ROCE_EQC_SHIFT_S, - eq->shift); - - /* set eq MSI_IDX */ - roce_set_field(eqc->byte_28, HNS_ROCE_EQC_MSI_INDX_M, - HNS_ROCE_EQC_MSI_INDX_S, HNS_ROCE_EQ_INIT_MSI_IDX); - - /* set cur_eqe_ba [27:12] */ - roce_set_field(eqc->byte_28, HNS_ROCE_EQC_CUR_EQE_BA_L_M, - HNS_ROCE_EQC_CUR_EQE_BA_L_S, eqe_ba[0] >> 12); - - /* set cur_eqe_ba [59:28] */ - roce_set_field(eqc->byte_32, HNS_ROCE_EQC_CUR_EQE_BA_M_M, - HNS_ROCE_EQC_CUR_EQE_BA_M_S, eqe_ba[0] >> 28); - - /* set cur_eqe_ba [63:60] */ - roce_set_field(eqc->byte_36, HNS_ROCE_EQC_CUR_EQE_BA_H_M, - HNS_ROCE_EQC_CUR_EQE_BA_H_S, eqe_ba[0] >> 60); - - /* set eq consumer idx */ - roce_set_field(eqc->byte_36, HNS_ROCE_EQC_CONS_INDX_M, - HNS_ROCE_EQC_CONS_INDX_S, HNS_ROCE_EQ_INIT_CONS_IDX); - - roce_set_field(eqc->byte_40, HNS_ROCE_EQC_NXT_EQE_BA_L_M, - HNS_ROCE_EQC_NXT_EQE_BA_L_S, eqe_ba[1] >> 12); - - roce_set_field(eqc->byte_44, HNS_ROCE_EQC_NXT_EQE_BA_H_M, - HNS_ROCE_EQC_NXT_EQE_BA_H_S, eqe_ba[1] >> 44); - - roce_set_field(eqc->byte_44, HNS_ROCE_EQC_EQE_SIZE_M, - HNS_ROCE_EQC_EQE_SIZE_S, - eq->eqe_size == HNS_ROCE_V3_EQE_SIZE ? 1 : 0); + hr_reg_write(eqc, EQC_EQ_ST, HNS_ROCE_V2_EQ_STATE_VALID); + hr_reg_write(eqc, EQC_EQE_HOP_NUM, eq->hop_num); + hr_reg_write(eqc, EQC_OVER_IGNORE, eq->over_ignore); + hr_reg_write(eqc, EQC_COALESCE, eq->coalesce); + hr_reg_write(eqc, EQC_ARM_ST, eq->arm_st); + hr_reg_write(eqc, EQC_EQN, eq->eqn); + hr_reg_write(eqc, EQC_EQE_CNT, HNS_ROCE_EQ_INIT_EQE_CNT); + hr_reg_write(eqc, EQC_EQE_BA_PG_SZ, + to_hr_hw_page_shift(eq->mtr.hem_cfg.ba_pg_shift)); + hr_reg_write(eqc, EQC_EQE_BUF_PG_SZ, + to_hr_hw_page_shift(eq->mtr.hem_cfg.buf_pg_shift)); + hr_reg_write(eqc, EQC_EQ_PROD_INDX, HNS_ROCE_EQ_INIT_PROD_IDX); + hr_reg_write(eqc, EQC_EQ_MAX_CNT, eq->eq_max_cnt); + + hr_reg_write(eqc, EQC_EQ_PERIOD, eq->eq_period); + hr_reg_write(eqc, EQC_EQE_REPORT_TIMER, HNS_ROCE_EQ_INIT_REPORT_TIMER); + hr_reg_write(eqc, EQC_EQE_BA_L, bt_ba >> 3); + hr_reg_write(eqc, EQC_EQE_BA_H, bt_ba >> 35); + hr_reg_write(eqc, EQC_SHIFT, eq->shift); + hr_reg_write(eqc, EQC_MSI_INDX, HNS_ROCE_EQ_INIT_MSI_IDX); + hr_reg_write(eqc, EQC_CUR_EQE_BA_L, eqe_ba[0] >> 12); + hr_reg_write(eqc, EQC_CUR_EQE_BA_M, eqe_ba[0] >> 28); + hr_reg_write(eqc, EQC_CUR_EQE_BA_H, eqe_ba[0] >> 60); + hr_reg_write(eqc, EQC_EQ_CONS_INDX, HNS_ROCE_EQ_INIT_CONS_IDX); + hr_reg_write(eqc, EQC_NEX_EQE_BA_L, eqe_ba[1] >> 12); + hr_reg_write(eqc, EQC_NEX_EQE_BA_H, eqe_ba[1] >> 44); + hr_reg_write(eqc, EQC_EQE_SIZE, + !!(eq->eqe_size == HNS_ROCE_V3_EQE_SIZE)); return 0; } @@ -6166,6 +6499,7 @@ static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev) hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE); __hns_roce_free_irq(hr_dev); + destroy_workqueue(hr_dev->irq_workq); for (i = 0; i < eq_num; i++) { hns_roce_v2_destroy_eqc(hr_dev, i); @@ -6174,9 +6508,6 @@ static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev) } kfree(eq_table->eq); - - flush_workqueue(hr_dev->irq_workq); - destroy_workqueue(hr_dev->irq_workq); } static const struct hns_roce_dfx_hw hns_roce_dfx_hw_v2 = { @@ -6205,9 +6536,9 @@ static const struct hns_roce_hw hns_roce_hw_v2 = { .hw_profile = hns_roce_v2_profile, .hw_init = hns_roce_v2_init, .hw_exit = hns_roce_v2_exit, - .post_mbox = hns_roce_v2_post_mbox, - .chk_mbox = hns_roce_v2_chk_mbox, - .rst_prc_mbox = hns_roce_v2_rst_process_cmd, + .post_mbox = v2_post_mbox, + .poll_mbox_done = v2_poll_mbox_done, + .chk_mbox_avail = v2_chk_mbox_is_avail, .set_gid = hns_roce_v2_set_gid, .set_mac = hns_roce_v2_set_mac, .write_mtpt = hns_roce_v2_write_mtpt, @@ -6218,20 +6549,10 @@ static const struct hns_roce_hw hns_roce_hw_v2 = { .set_hem = hns_roce_v2_set_hem, .clear_hem = hns_roce_v2_clear_hem, .modify_qp = hns_roce_v2_modify_qp, - .query_qp = hns_roce_v2_query_qp, - .destroy_qp = hns_roce_v2_destroy_qp, .qp_flow_control_init = hns_roce_v2_qp_flow_control_init, - .modify_cq = hns_roce_v2_modify_cq, - .post_send = hns_roce_v2_post_send, - .post_recv = hns_roce_v2_post_recv, - .req_notify_cq = hns_roce_v2_req_notify_cq, - .poll_cq = hns_roce_v2_poll_cq, .init_eq = hns_roce_v2_init_eq_table, .cleanup_eq = hns_roce_v2_cleanup_eq_table, .write_srqc = hns_roce_v2_write_srqc, - .modify_srq = hns_roce_v2_modify_srq, - .query_srq = hns_roce_v2_query_srq, - .post_srq_recv = hns_roce_v2_post_srq_recv, .hns_roce_dev_ops = &hns_roce_v2_dev_ops, .hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops, }; @@ -6243,6 +6564,8 @@ static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = { {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0}, {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0}, {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA), 0}, + {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF), + HNAE3_DEV_SUPPORT_ROCE_DCB_BITS}, /* required last entry */ {0, } }; @@ -6253,9 +6576,12 @@ static void hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev, struct hnae3_handle *handle) { struct hns_roce_v2_priv *priv = hr_dev->priv; + const struct pci_device_id *id; int i; hr_dev->pci_dev = handle->pdev; + id = pci_match_id(hns_roce_hw_v2_pci_tbl, hr_dev->pci_dev); + hr_dev->is_vf = id->driver_data; hr_dev->dev = &handle->pdev->dev; hr_dev->hw = &hns_roce_hw_v2; hr_dev->dfx = &hns_roce_dfx_hw_v2; @@ -6272,7 +6598,7 @@ static void hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev, addrconf_addr_eui48((u8 *)&hr_dev->ib_dev.node_guid, hr_dev->iboe.netdevs[0]->dev_addr); - for (i = 0; i < HNS_ROCE_V2_MAX_IRQ_NUM; i++) + for (i = 0; i < handle->rinfo.num_vectors; i++) hr_dev->irq[i] = pci_irq_vector(handle->pdev, i + handle->rinfo.base_vector); @@ -6356,6 +6682,9 @@ static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle) if (!id) return 0; + if (id->driver_data && handle->pdev->revision < PCI_REVISION_ID_HIP09) + return 0; + ret = __hns_roce_hw_v2_init_instance(handle); if (ret) { handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index 39621fb6ec16..a2100a629859 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -40,13 +40,11 @@ #define HNS_ROCE_VF_SRQC_BT_NUM 64 #define HNS_ROCE_VF_CQC_BT_NUM 64 #define HNS_ROCE_VF_MPT_BT_NUM 64 -#define HNS_ROCE_VF_EQC_NUM 64 #define HNS_ROCE_VF_SMAC_NUM 32 -#define HNS_ROCE_VF_SGID_NUM 32 #define HNS_ROCE_VF_SL_NUM 8 #define HNS_ROCE_VF_GMV_BT_NUM 256 -#define HNS_ROCE_V2_MAX_QP_NUM 0x100000 +#define HNS_ROCE_V2_MAX_QP_NUM 0x1000 #define HNS_ROCE_V2_MAX_QPC_TIMER_NUM 0x200 #define HNS_ROCE_V2_MAX_WQE_NUM 0x8000 #define HNS_ROCE_V2_MAX_SRQ 0x100000 @@ -61,6 +59,7 @@ #define HNS_ROCE_V2_MAX_SQ_SGE_NUM 64 #define HNS_ROCE_V2_MAX_EXTEND_SGE_NUM 0x200000 #define HNS_ROCE_V2_MAX_SQ_INLINE 0x20 +#define HNS_ROCE_V2_MAX_SQ_INL_EXT 0x400 #define HNS_ROCE_V2_MAX_RC_INL_INN_SZ 32 #define HNS_ROCE_V2_UAR_NUM 256 #define HNS_ROCE_V2_PHY_UAR_NUM 1 @@ -74,6 +73,8 @@ #define HNS_ROCE_V2_MAX_SRQWQE_SEGS 0x1000000 #define HNS_ROCE_V2_MAX_IDX_SEGS 0x1000000 #define HNS_ROCE_V2_MAX_PD_NUM 0x1000000 +#define HNS_ROCE_V2_MAX_XRCD_NUM 0x1000000 +#define HNS_ROCE_V2_RSV_XRCD_NUM 0 #define HNS_ROCE_V2_MAX_QP_INIT_RDMA 128 #define HNS_ROCE_V2_MAX_QP_DEST_RDMA 128 #define HNS_ROCE_V2_MAX_SQ_DESC_SZ 64 @@ -121,7 +122,7 @@ #define HNS_ROCE_BA_PG_SZ_SUPPORTED_256K 6 #define HNS_ROCE_BA_PG_SZ_SUPPORTED_16K 2 -#define HNS_ROCE_V2_GID_INDEX_NUM 256 +#define HNS_ROCE_V2_GID_INDEX_NUM 16 #define HNS_ROCE_V2_TABLE_CHUNK_SIZE (1 << 18) @@ -143,6 +144,8 @@ #define HNS_ROCE_CMQ_SCC_CLR_DONE_CNT 5 +#define HNS_ROCE_CONG_SIZE 64 + #define check_whether_last_step(hop_num, step_idx) \ ((step_idx == 0 && hop_num == HNS_ROCE_HOP_NUM_0) || \ (step_idx == 1 && hop_num == 1) || \ @@ -195,11 +198,11 @@ enum { }; enum { - HNS_ROCE_V2_SQ_DB = 0x0, - HNS_ROCE_V2_RQ_DB = 0x1, - HNS_ROCE_V2_SRQ_DB = 0x2, - HNS_ROCE_V2_CQ_DB_PTR = 0x3, - HNS_ROCE_V2_CQ_DB_NTR = 0x4, + HNS_ROCE_V2_SQ_DB, + HNS_ROCE_V2_RQ_DB, + HNS_ROCE_V2_SRQ_DB, + HNS_ROCE_V2_CQ_DB, + HNS_ROCE_V2_CQ_DB_NOTIFY }; enum { @@ -233,6 +236,7 @@ enum hns_roce_opcode_type { HNS_ROCE_OPC_CFG_EXT_LLM = 0x8403, HNS_ROCE_OPC_CFG_TMOUT_LLM = 0x8404, HNS_ROCE_OPC_QUERY_PF_TIMER_RES = 0x8406, + HNS_ROCE_OPC_QUERY_FUNC_INFO = 0x8407, HNS_ROCE_OPC_QUERY_PF_CAPS_NUM = 0x8408, HNS_ROCE_OPC_CFG_ENTRY_SIZE = 0x8409, HNS_ROCE_OPC_CFG_SGID_TB = 0x8500, @@ -244,6 +248,7 @@ enum hns_roce_opcode_type { HNS_ROCE_OPC_CLR_SCCC = 0x8509, HNS_ROCE_OPC_QUERY_SCCC = 0x850a, HNS_ROCE_OPC_RESET_SCCC = 0x850b, + HNS_ROCE_OPC_QUERY_VF_RES = 0x850e, HNS_ROCE_OPC_CFG_GMV_TBL = 0x850f, HNS_ROCE_OPC_CFG_GMV_BT = 0x8510, HNS_SWITCH_PARAMETER_CFG = 0x1033, @@ -255,10 +260,20 @@ enum { }; enum hns_roce_cmd_return_status { - CMD_EXEC_SUCCESS = 0, - CMD_NO_AUTH = 1, - CMD_NOT_EXEC = 2, - CMD_QUEUE_FULL = 3, + CMD_EXEC_SUCCESS, + CMD_NO_AUTH, + CMD_NOT_EXIST, + CMD_CRQ_FULL, + CMD_NEXT_ERR, + CMD_NOT_EXEC, + CMD_PARA_ERR, + CMD_RESULT_ERR, + CMD_TIMEOUT, + CMD_HILINK_ERR, + CMD_INFO_ILLEGAL, + CMD_INVALID, + CMD_ROH_CHECK_FAIL, + CMD_OTHER_ERR = 0xff }; enum hns_roce_sgid_type { @@ -399,7 +414,8 @@ struct hns_roce_srq_context { #define SRQC_CONSUMER_IDX SRQC_FIELD_LOC(127, 112) #define SRQC_WQE_BT_BA_L SRQC_FIELD_LOC(159, 128) #define SRQC_WQE_BT_BA_H SRQC_FIELD_LOC(188, 160) -#define SRQC_RSV2 SRQC_FIELD_LOC(191, 189) +#define SRQC_RSV2 SRQC_FIELD_LOC(190, 189) +#define SRQC_SRQ_TYPE SRQC_FIELD_LOC(191, 191) #define SRQC_PD SRQC_FIELD_LOC(215, 192) #define SRQC_RQWS SRQC_FIELD_LOC(219, 216) #define SRQC_RSV3 SRQC_FIELD_LOC(223, 220) @@ -572,6 +588,10 @@ struct hns_roce_v2_qp_context { struct hns_roce_v2_qp_context_ex ext; }; +#define QPC_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_v2_qp_context, h, l) + +#define QPC_CONG_ALGO_TMPL_ID QPC_FIELD_LOC(455, 448) + #define V2_QPC_BYTE_4_TST_S 0 #define V2_QPC_BYTE_4_TST_M GENMASK(2, 0) @@ -663,9 +683,6 @@ struct hns_roce_v2_qp_context { #define V2_QPC_BYTE_56_LP_PKTN_INI_S 28 #define V2_QPC_BYTE_56_LP_PKTN_INI_M GENMASK(31, 28) -#define V2_QPC_BYTE_60_TEMPID_S 0 -#define V2_QPC_BYTE_60_TEMPID_M GENMASK(7, 0) - #define V2_QPC_BYTE_60_SCC_TOKEN_S 8 #define V2_QPC_BYTE_60_SCC_TOKEN_M GENMASK(26, 8) @@ -698,6 +715,8 @@ struct hns_roce_v2_qp_context { #define V2_QPC_BYTE_80_RX_CQN_S 0 #define V2_QPC_BYTE_80_RX_CQN_M GENMASK(23, 0) +#define V2_QPC_BYTE_80_XRC_QP_TYPE_S 24 + #define V2_QPC_BYTE_80_MIN_RNR_TIME_S 27 #define V2_QPC_BYTE_80_MIN_RNR_TIME_M GENMASK(31, 27) @@ -940,6 +959,10 @@ struct hns_roce_v2_qp_context { #define QPCEX_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_v2_qp_context_ex, h, l) +#define QPCEX_CONG_ALG_SEL QPCEX_FIELD_LOC(0, 0) +#define QPCEX_CONG_ALG_SUB_SEL QPCEX_FIELD_LOC(1, 1) +#define QPCEX_DIP_CTX_IDX_VLD QPCEX_FIELD_LOC(2, 2) +#define QPCEX_DIP_CTX_IDX QPCEX_FIELD_LOC(22, 3) #define QPCEX_STASH QPCEX_FIELD_LOC(82, 82) #define V2_QP_RWE_S 1 /* rdma write enable */ @@ -1130,33 +1153,27 @@ struct hns_roce_v2_mpt_entry { #define V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S 28 #define V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M GENMASK(31, 28) -#define V2_DB_BYTE_4_TAG_S 0 -#define V2_DB_BYTE_4_TAG_M GENMASK(23, 0) +#define V2_DB_TAG_S 0 +#define V2_DB_TAG_M GENMASK(23, 0) -#define V2_DB_BYTE_4_CMD_S 24 -#define V2_DB_BYTE_4_CMD_M GENMASK(27, 24) +#define V2_DB_CMD_S 24 +#define V2_DB_CMD_M GENMASK(27, 24) #define V2_DB_FLAG_S 31 -#define V2_DB_PARAMETER_IDX_S 0 -#define V2_DB_PARAMETER_IDX_M GENMASK(15, 0) +#define V2_DB_PRODUCER_IDX_S 0 +#define V2_DB_PRODUCER_IDX_M GENMASK(15, 0) -#define V2_DB_PARAMETER_SL_S 16 -#define V2_DB_PARAMETER_SL_M GENMASK(18, 16) +#define V2_DB_SL_S 16 +#define V2_DB_SL_M GENMASK(18, 16) -#define V2_CQ_DB_BYTE_4_TAG_S 0 -#define V2_CQ_DB_BYTE_4_TAG_M GENMASK(23, 0) +#define V2_CQ_DB_CONS_IDX_S 0 +#define V2_CQ_DB_CONS_IDX_M GENMASK(23, 0) -#define V2_CQ_DB_BYTE_4_CMD_S 24 -#define V2_CQ_DB_BYTE_4_CMD_M GENMASK(27, 24) +#define V2_CQ_DB_NOTIFY_TYPE_S 24 -#define V2_CQ_DB_PARAMETER_CONS_IDX_S 0 -#define V2_CQ_DB_PARAMETER_CONS_IDX_M GENMASK(23, 0) - -#define V2_CQ_DB_PARAMETER_CMD_SN_S 25 -#define V2_CQ_DB_PARAMETER_CMD_SN_M GENMASK(26, 25) - -#define V2_CQ_DB_PARAMETER_NOTIFY_S 24 +#define V2_CQ_DB_CMD_SN_S 25 +#define V2_CQ_DB_CMD_SN_M GENMASK(26, 25) struct hns_roce_v2_ud_send_wqe { __le32 byte_4; @@ -1359,194 +1376,44 @@ struct hns_roce_cfg_llm_b { #define CFG_LLM_TAIL_PTR_S 0 #define CFG_LLM_TAIL_PTR_M GENMASK(11, 0) -struct hns_roce_cfg_global_param { - __le32 time_cfg_udp_port; - __le32 rsv[5]; -}; - -#define CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_S 0 -#define CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_M GENMASK(9, 0) - -#define CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_S 16 -#define CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_M GENMASK(31, 16) - -struct hns_roce_pf_res_a { - __le32 rsv; - __le32 qpc_bt_idx_num; - __le32 srqc_bt_idx_num; - __le32 cqc_bt_idx_num; - __le32 mpt_bt_idx_num; - __le32 eqc_bt_idx_num; -}; - -#define PF_RES_DATA_1_PF_QPC_BT_IDX_S 0 -#define PF_RES_DATA_1_PF_QPC_BT_IDX_M GENMASK(10, 0) - -#define PF_RES_DATA_1_PF_QPC_BT_NUM_S 16 -#define PF_RES_DATA_1_PF_QPC_BT_NUM_M GENMASK(27, 16) - -#define PF_RES_DATA_2_PF_SRQC_BT_IDX_S 0 -#define PF_RES_DATA_2_PF_SRQC_BT_IDX_M GENMASK(8, 0) - -#define PF_RES_DATA_2_PF_SRQC_BT_NUM_S 16 -#define PF_RES_DATA_2_PF_SRQC_BT_NUM_M GENMASK(25, 16) - -#define PF_RES_DATA_3_PF_CQC_BT_IDX_S 0 -#define PF_RES_DATA_3_PF_CQC_BT_IDX_M GENMASK(8, 0) - -#define PF_RES_DATA_3_PF_CQC_BT_NUM_S 16 -#define PF_RES_DATA_3_PF_CQC_BT_NUM_M GENMASK(25, 16) - -#define PF_RES_DATA_4_PF_MPT_BT_IDX_S 0 -#define PF_RES_DATA_4_PF_MPT_BT_IDX_M GENMASK(8, 0) - -#define PF_RES_DATA_4_PF_MPT_BT_NUM_S 16 -#define PF_RES_DATA_4_PF_MPT_BT_NUM_M GENMASK(25, 16) - -#define PF_RES_DATA_5_PF_EQC_BT_IDX_S 0 -#define PF_RES_DATA_5_PF_EQC_BT_IDX_M GENMASK(8, 0) - -#define PF_RES_DATA_5_PF_EQC_BT_NUM_S 16 -#define PF_RES_DATA_5_PF_EQC_BT_NUM_M GENMASK(25, 16) - -struct hns_roce_pf_res_b { - __le32 rsv0; - __le32 smac_idx_num; - __le32 sgid_idx_num; - __le32 qid_idx_sl_num; - __le32 sccc_bt_idx_num; - __le32 gmv_idx_num; -}; - -#define PF_RES_DATA_1_PF_SMAC_IDX_S 0 -#define PF_RES_DATA_1_PF_SMAC_IDX_M GENMASK(7, 0) - -#define PF_RES_DATA_1_PF_SMAC_NUM_S 8 -#define PF_RES_DATA_1_PF_SMAC_NUM_M GENMASK(16, 8) - -#define PF_RES_DATA_2_PF_SGID_IDX_S 0 -#define PF_RES_DATA_2_PF_SGID_IDX_M GENMASK(7, 0) - -#define PF_RES_DATA_2_PF_SGID_NUM_S 8 -#define PF_RES_DATA_2_PF_SGID_NUM_M GENMASK(16, 8) - -#define PF_RES_DATA_3_PF_QID_IDX_S 0 -#define PF_RES_DATA_3_PF_QID_IDX_M GENMASK(9, 0) - -#define PF_RES_DATA_3_PF_SL_NUM_S 16 -#define PF_RES_DATA_3_PF_SL_NUM_M GENMASK(26, 16) - -#define PF_RES_DATA_4_PF_SCCC_BT_IDX_S 0 -#define PF_RES_DATA_4_PF_SCCC_BT_IDX_M GENMASK(8, 0) - -#define PF_RES_DATA_4_PF_SCCC_BT_NUM_S 9 -#define PF_RES_DATA_4_PF_SCCC_BT_NUM_M GENMASK(17, 9) - -#define PF_RES_DATA_5_PF_GMV_BT_IDX_S 0 -#define PF_RES_DATA_5_PF_GMV_BT_IDX_M GENMASK(7, 0) +/* Fields of HNS_ROCE_OPC_CFG_GLOBAL_PARAM */ +#define CFG_GLOBAL_PARAM_1US_CYCLES CMQ_REQ_FIELD_LOC(9, 0) +#define CFG_GLOBAL_PARAM_UDP_PORT CMQ_REQ_FIELD_LOC(31, 16) -#define PF_RES_DATA_5_PF_GMV_BT_NUM_S 8 -#define PF_RES_DATA_5_PF_GMV_BT_NUM_M GENMASK(16, 8) - -struct hns_roce_pf_timer_res_a { - __le32 rsv0; - __le32 qpc_timer_bt_idx_num; - __le32 cqc_timer_bt_idx_num; - __le32 rsv[3]; -}; - -#define PF_RES_DATA_1_PF_QPC_TIMER_BT_IDX_S 0 -#define PF_RES_DATA_1_PF_QPC_TIMER_BT_IDX_M GENMASK(11, 0) - -#define PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_S 16 -#define PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_M GENMASK(28, 16) - -#define PF_RES_DATA_2_PF_CQC_TIMER_BT_IDX_S 0 -#define PF_RES_DATA_2_PF_CQC_TIMER_BT_IDX_M GENMASK(10, 0) - -#define PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_S 16 -#define PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_M GENMASK(27, 16) - -struct hns_roce_vf_res_a { - __le32 vf_id; - __le32 vf_qpc_bt_idx_num; - __le32 vf_srqc_bt_idx_num; - __le32 vf_cqc_bt_idx_num; - __le32 vf_mpt_bt_idx_num; - __le32 vf_eqc_bt_idx_num; -}; - -#define VF_RES_A_DATA_1_VF_QPC_BT_IDX_S 0 -#define VF_RES_A_DATA_1_VF_QPC_BT_IDX_M GENMASK(10, 0) - -#define VF_RES_A_DATA_1_VF_QPC_BT_NUM_S 16 -#define VF_RES_A_DATA_1_VF_QPC_BT_NUM_M GENMASK(27, 16) - -#define VF_RES_A_DATA_2_VF_SRQC_BT_IDX_S 0 -#define VF_RES_A_DATA_2_VF_SRQC_BT_IDX_M GENMASK(8, 0) - -#define VF_RES_A_DATA_2_VF_SRQC_BT_NUM_S 16 -#define VF_RES_A_DATA_2_VF_SRQC_BT_NUM_M GENMASK(25, 16) - -#define VF_RES_A_DATA_3_VF_CQC_BT_IDX_S 0 -#define VF_RES_A_DATA_3_VF_CQC_BT_IDX_M GENMASK(8, 0) - -#define VF_RES_A_DATA_3_VF_CQC_BT_NUM_S 16 -#define VF_RES_A_DATA_3_VF_CQC_BT_NUM_M GENMASK(25, 16) - -#define VF_RES_A_DATA_4_VF_MPT_BT_IDX_S 0 -#define VF_RES_A_DATA_4_VF_MPT_BT_IDX_M GENMASK(8, 0) - -#define VF_RES_A_DATA_4_VF_MPT_BT_NUM_S 16 -#define VF_RES_A_DATA_4_VF_MPT_BT_NUM_M GENMASK(25, 16) - -#define VF_RES_A_DATA_5_VF_EQC_IDX_S 0 -#define VF_RES_A_DATA_5_VF_EQC_IDX_M GENMASK(8, 0) - -#define VF_RES_A_DATA_5_VF_EQC_NUM_S 16 -#define VF_RES_A_DATA_5_VF_EQC_NUM_M GENMASK(25, 16) - -struct hns_roce_vf_res_b { - __le32 rsv0; - __le32 vf_smac_idx_num; - __le32 vf_sgid_idx_num; - __le32 vf_qid_idx_sl_num; - __le32 vf_sccc_idx_num; - __le32 vf_gmv_idx_num; -}; - -#define VF_RES_B_DATA_0_VF_ID_S 0 -#define VF_RES_B_DATA_0_VF_ID_M GENMASK(7, 0) - -#define VF_RES_B_DATA_1_VF_SMAC_IDX_S 0 -#define VF_RES_B_DATA_1_VF_SMAC_IDX_M GENMASK(7, 0) - -#define VF_RES_B_DATA_1_VF_SMAC_NUM_S 8 -#define VF_RES_B_DATA_1_VF_SMAC_NUM_M GENMASK(16, 8) - -#define VF_RES_B_DATA_2_VF_SGID_IDX_S 0 -#define VF_RES_B_DATA_2_VF_SGID_IDX_M GENMASK(7, 0) - -#define VF_RES_B_DATA_2_VF_SGID_NUM_S 8 -#define VF_RES_B_DATA_2_VF_SGID_NUM_M GENMASK(16, 8) - -#define VF_RES_B_DATA_3_VF_QID_IDX_S 0 -#define VF_RES_B_DATA_3_VF_QID_IDX_M GENMASK(9, 0) - -#define VF_RES_B_DATA_3_VF_SL_NUM_S 16 -#define VF_RES_B_DATA_3_VF_SL_NUM_M GENMASK(19, 16) - -#define VF_RES_B_DATA_4_VF_SCCC_BT_IDX_S 0 -#define VF_RES_B_DATA_4_VF_SCCC_BT_IDX_M GENMASK(8, 0) - -#define VF_RES_B_DATA_4_VF_SCCC_BT_NUM_S 9 -#define VF_RES_B_DATA_4_VF_SCCC_BT_NUM_M GENMASK(17, 9) - -#define VF_RES_B_DATA_5_VF_GMV_BT_IDX_S 0 -#define VF_RES_B_DATA_5_VF_GMV_BT_IDX_M GENMASK(7, 0) - -#define VF_RES_B_DATA_5_VF_GMV_BT_NUM_S 16 -#define VF_RES_B_DATA_5_VF_GMV_BT_NUM_M GENMASK(24, 16) +/* + * Fields of HNS_ROCE_OPC_QUERY_PF_RES, HNS_ROCE_OPC_QUERY_VF_RES + * and HNS_ROCE_OPC_ALLOC_VF_RES + */ +#define FUNC_RES_A_VF_ID CMQ_REQ_FIELD_LOC(7, 0) +#define FUNC_RES_A_QPC_BT_IDX CMQ_REQ_FIELD_LOC(42, 32) +#define FUNC_RES_A_QPC_BT_NUM CMQ_REQ_FIELD_LOC(59, 48) +#define FUNC_RES_A_SRQC_BT_IDX CMQ_REQ_FIELD_LOC(72, 64) +#define FUNC_RES_A_SRQC_BT_NUM CMQ_REQ_FIELD_LOC(89, 80) +#define FUNC_RES_A_CQC_BT_IDX CMQ_REQ_FIELD_LOC(104, 96) +#define FUNC_RES_A_CQC_BT_NUM CMQ_REQ_FIELD_LOC(121, 112) +#define FUNC_RES_A_MPT_BT_IDX CMQ_REQ_FIELD_LOC(136, 128) +#define FUNC_RES_A_MPT_BT_NUM CMQ_REQ_FIELD_LOC(153, 144) +#define FUNC_RES_A_EQC_BT_IDX CMQ_REQ_FIELD_LOC(168, 160) +#define FUNC_RES_A_EQC_BT_NUM CMQ_REQ_FIELD_LOC(185, 176) +#define FUNC_RES_B_SMAC_IDX CMQ_REQ_FIELD_LOC(39, 32) +#define FUNC_RES_B_SMAC_NUM CMQ_REQ_FIELD_LOC(48, 40) +#define FUNC_RES_B_SGID_IDX CMQ_REQ_FIELD_LOC(71, 64) +#define FUNC_RES_B_SGID_NUM CMQ_REQ_FIELD_LOC(80, 72) +#define FUNC_RES_B_QID_IDX CMQ_REQ_FIELD_LOC(105, 96) +#define FUNC_RES_B_QID_NUM CMQ_REQ_FIELD_LOC(122, 112) +#define FUNC_RES_V_QID_NUM CMQ_REQ_FIELD_LOC(115, 112) + +#define FUNC_RES_B_SCCC_BT_IDX CMQ_REQ_FIELD_LOC(136, 128) +#define FUNC_RES_B_SCCC_BT_NUM CMQ_REQ_FIELD_LOC(145, 137) +#define FUNC_RES_B_GMV_BT_IDX CMQ_REQ_FIELD_LOC(167, 160) +#define FUNC_RES_B_GMV_BT_NUM CMQ_REQ_FIELD_LOC(176, 168) +#define FUNC_RES_V_GMV_BT_NUM CMQ_REQ_FIELD_LOC(184, 176) + +/* Fields of HNS_ROCE_OPC_QUERY_PF_TIMER_RES */ +#define PF_TIMER_RES_QPC_ITEM_IDX CMQ_REQ_FIELD_LOC(43, 32) +#define PF_TIMER_RES_QPC_ITEM_NUM CMQ_REQ_FIELD_LOC(60, 48) +#define PF_TIMER_RES_CQC_ITEM_IDX CMQ_REQ_FIELD_LOC(74, 64) +#define PF_TIMER_RES_CQC_ITEM_NUM CMQ_REQ_FIELD_LOC(91, 80) struct hns_roce_vf_switch { __le32 rocee_sel; @@ -1578,59 +1445,43 @@ struct hns_roce_mbox_status { __le32 rsv[5]; }; -struct hns_roce_cfg_bt_attr { - __le32 vf_qpc_cfg; - __le32 vf_srqc_cfg; - __le32 vf_cqc_cfg; - __le32 vf_mpt_cfg; - __le32 vf_sccc_cfg; - __le32 rsv; +#define HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS 10000 + +#define MB_ST_HW_RUN_M BIT(31) +#define MB_ST_COMPLETE_M GENMASK(7, 0) + +#define MB_ST_COMPLETE_SUCC 1 + +/* Fields of HNS_ROCE_OPC_CFG_BT_ATTR */ +#define CFG_BT_ATTR_QPC_BA_PGSZ CMQ_REQ_FIELD_LOC(3, 0) +#define CFG_BT_ATTR_QPC_BUF_PGSZ CMQ_REQ_FIELD_LOC(7, 4) +#define CFG_BT_ATTR_QPC_HOPNUM CMQ_REQ_FIELD_LOC(9, 8) +#define CFG_BT_ATTR_SRQC_BA_PGSZ CMQ_REQ_FIELD_LOC(35, 32) +#define CFG_BT_ATTR_SRQC_BUF_PGSZ CMQ_REQ_FIELD_LOC(39, 36) +#define CFG_BT_ATTR_SRQC_HOPNUM CMQ_REQ_FIELD_LOC(41, 40) +#define CFG_BT_ATTR_CQC_BA_PGSZ CMQ_REQ_FIELD_LOC(67, 64) +#define CFG_BT_ATTR_CQC_BUF_PGSZ CMQ_REQ_FIELD_LOC(71, 68) +#define CFG_BT_ATTR_CQC_HOPNUM CMQ_REQ_FIELD_LOC(73, 72) +#define CFG_BT_ATTR_MPT_BA_PGSZ CMQ_REQ_FIELD_LOC(99, 96) +#define CFG_BT_ATTR_MPT_BUF_PGSZ CMQ_REQ_FIELD_LOC(103, 100) +#define CFG_BT_ATTR_MPT_HOPNUM CMQ_REQ_FIELD_LOC(105, 104) +#define CFG_BT_ATTR_SCCC_BA_PGSZ CMQ_REQ_FIELD_LOC(131, 128) +#define CFG_BT_ATTR_SCCC_BUF_PGSZ CMQ_REQ_FIELD_LOC(135, 132) +#define CFG_BT_ATTR_SCCC_HOPNUM CMQ_REQ_FIELD_LOC(137, 136) + +/* Fields of HNS_ROCE_OPC_CFG_ENTRY_SIZE */ +#define CFG_HEM_ENTRY_SIZE_TYPE CMQ_REQ_FIELD_LOC(31, 0) +enum { + HNS_ROCE_CFG_QPC_SIZE = BIT(0), + HNS_ROCE_CFG_SCCC_SIZE = BIT(1), }; -#define CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_S 0 -#define CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_M GENMASK(3, 0) - -#define CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_S 4 -#define CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_M GENMASK(7, 4) - -#define CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_S 8 -#define CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_M GENMASK(9, 8) - -#define CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_S 0 -#define CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_M GENMASK(3, 0) - -#define CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_S 4 -#define CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_M GENMASK(7, 4) - -#define CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_S 8 -#define CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_M GENMASK(9, 8) - -#define CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_S 0 -#define CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_M GENMASK(3, 0) - -#define CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_S 4 -#define CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_M GENMASK(7, 4) - -#define CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_S 8 -#define CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_M GENMASK(9, 8) - -#define CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_S 0 -#define CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_M GENMASK(3, 0) - -#define CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_S 4 -#define CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_M GENMASK(7, 4) - -#define CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_S 8 -#define CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_M GENMASK(9, 8) - -#define CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_S 0 -#define CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_M GENMASK(3, 0) - -#define CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_S 4 -#define CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_M GENMASK(7, 4) +#define CFG_HEM_ENTRY_SIZE_VALUE CMQ_REQ_FIELD_LOC(191, 160) -#define CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_S 8 -#define CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_M GENMASK(9, 8) +/* Fields of HNS_ROCE_OPC_CFG_GMV_BT */ +#define CFG_GMV_BT_BA_L CMQ_REQ_FIELD_LOC(31, 0) +#define CFG_GMV_BT_BA_H CMQ_REQ_FIELD_LOC(51, 32) +#define CFG_GMV_BT_IDX CMQ_REQ_FIELD_LOC(95, 64) struct hns_roce_cfg_sgid_tb { __le32 table_idx_rsv; @@ -1641,17 +1492,6 @@ struct hns_roce_cfg_sgid_tb { __le32 vf_sgid_type_rsv; }; -enum { - HNS_ROCE_CFG_QPC_SIZE = BIT(0), - HNS_ROCE_CFG_SCCC_SIZE = BIT(1), -}; - -struct hns_roce_cfg_entry_size { - __le32 type; - __le32 rsv[4]; - __le32 size; -}; - #define CFG_SGID_TB_TABLE_IDX_S 0 #define CFG_SGID_TB_TABLE_IDX_M GENMASK(7, 0) @@ -1670,16 +1510,6 @@ struct hns_roce_cfg_smac_tb { #define CFG_SMAC_TB_VF_SMAC_H_S 0 #define CFG_SMAC_TB_VF_SMAC_H_M GENMASK(15, 0) -struct hns_roce_cfg_gmv_bt { - __le32 gmv_ba_l; - __le32 gmv_ba_h; - __le32 gmv_bt_idx; - __le32 rsv[3]; -}; - -#define CFG_GMV_BA_H_S 0 -#define CFG_GMV_BA_H_M GENMASK(19, 0) - struct hns_roce_cfg_gmv_tb_a { __le32 vf_sgid_l; __le32 vf_sgid_ml; @@ -1805,6 +1635,14 @@ struct hns_roce_query_pf_caps_d { #define V2_QUERY_PF_CAPS_D_SQWQE_HOP_NUM_S 24 #define V2_QUERY_PF_CAPS_D_SQWQE_HOP_NUM_M GENMASK(25, 24) +#define V2_QUERY_PF_CAPS_D_CONG_TYPE_S 26 +#define V2_QUERY_PF_CAPS_D_CONG_TYPE_M GENMASK(29, 26) + +struct hns_roce_congestion_algorithm { + u8 alg_sel; + u8 alg_sub_sel; + u8 dip_vld; +}; #define V2_QUERY_PF_CAPS_D_CEQ_DEPTH_S 0 #define V2_QUERY_PF_CAPS_D_CEQ_DEPTH_M GENMASK(21, 0) @@ -1859,18 +1697,27 @@ struct hns_roce_query_pf_caps_e { #define V2_QUERY_PF_CAPS_E_RSV_LKEYS_S 0 #define V2_QUERY_PF_CAPS_E_RSV_LKEYS_M GENMASK(19, 0) +struct hns_roce_cmq_req { + __le32 data[6]; +}; + +#define CMQ_REQ_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_cmq_req, h, l) + struct hns_roce_cmq_desc { __le16 opcode; __le16 flag; __le16 retval; __le16 rsv; - __le32 data[6]; -}; - -#define HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS 10000 + union { + __le32 data[6]; + struct { + __le32 own_func_num; + __le32 own_mac_id; + __le32 rsv[4]; + } func_info; + }; -#define HNS_ROCE_HW_RUN_BIT_SHIFT 31 -#define HNS_ROCE_HW_MB_STATUS_MASK 0xFF +}; struct hns_roce_v2_cmq_ring { dma_addr_t desc_dma_addr; @@ -1932,6 +1779,12 @@ struct hns_roce_eq_context { __le32 rsv[5]; }; +struct hns_roce_dip { + u8 dgid[GID_LEN_V2]; + u8 dip_idx; + struct list_head node; /* all dips are on a list */ +}; + #define HNS_ROCE_AEQ_DEFAULT_BURST_NUM 0x0 #define HNS_ROCE_AEQ_DEFAULT_INTERVAL 0x0 #define HNS_ROCE_CEQ_DEFAULT_BURST_NUM 0x0 @@ -1966,8 +1819,7 @@ struct hns_roce_eq_context { #define HNS_ROCE_V2_ASYNC_EQE_NUM 0x1000 #define HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S 0 -#define HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S 1 -#define HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S 2 +#define HNS_ROCE_V2_VF_INT_ST_RAS_INT_S 1 #define HNS_ROCE_EQ_DB_CMD_AEQ 0x0 #define HNS_ROCE_EQ_DB_CMD_AEQ_ARMED 0x1 @@ -1982,96 +1834,38 @@ struct hns_roce_eq_context { #define HNS_ROCE_INT_NAME_LEN 32 #define HNS_ROCE_V2_EQN_M GENMASK(23, 0) -#define HNS_ROCE_V2_CONS_IDX_M GENMASK(23, 0) - #define HNS_ROCE_V2_VF_ABN_INT_EN_S 0 #define HNS_ROCE_V2_VF_ABN_INT_EN_M GENMASK(0, 0) #define HNS_ROCE_V2_VF_ABN_INT_ST_M GENMASK(2, 0) #define HNS_ROCE_V2_VF_ABN_INT_CFG_M GENMASK(2, 0) #define HNS_ROCE_V2_VF_EVENT_INT_EN_M GENMASK(0, 0) -/* WORD0 */ -#define HNS_ROCE_EQC_EQ_ST_S 0 -#define HNS_ROCE_EQC_EQ_ST_M GENMASK(1, 0) - -#define HNS_ROCE_EQC_HOP_NUM_S 2 -#define HNS_ROCE_EQC_HOP_NUM_M GENMASK(3, 2) - -#define HNS_ROCE_EQC_OVER_IGNORE_S 4 -#define HNS_ROCE_EQC_OVER_IGNORE_M GENMASK(4, 4) - -#define HNS_ROCE_EQC_COALESCE_S 5 -#define HNS_ROCE_EQC_COALESCE_M GENMASK(5, 5) - -#define HNS_ROCE_EQC_ARM_ST_S 6 -#define HNS_ROCE_EQC_ARM_ST_M GENMASK(7, 6) - -#define HNS_ROCE_EQC_EQN_S 8 -#define HNS_ROCE_EQC_EQN_M GENMASK(15, 8) - -#define HNS_ROCE_EQC_EQE_CNT_S 16 -#define HNS_ROCE_EQC_EQE_CNT_M GENMASK(31, 16) - -/* WORD1 */ -#define HNS_ROCE_EQC_BA_PG_SZ_S 0 -#define HNS_ROCE_EQC_BA_PG_SZ_M GENMASK(3, 0) - -#define HNS_ROCE_EQC_BUF_PG_SZ_S 4 -#define HNS_ROCE_EQC_BUF_PG_SZ_M GENMASK(7, 4) - -#define HNS_ROCE_EQC_PROD_INDX_S 8 -#define HNS_ROCE_EQC_PROD_INDX_M GENMASK(31, 8) - -/* WORD2 */ -#define HNS_ROCE_EQC_MAX_CNT_S 0 -#define HNS_ROCE_EQC_MAX_CNT_M GENMASK(15, 0) - -#define HNS_ROCE_EQC_PERIOD_S 16 -#define HNS_ROCE_EQC_PERIOD_M GENMASK(31, 16) - -/* WORD3 */ -#define HNS_ROCE_EQC_REPORT_TIMER_S 0 -#define HNS_ROCE_EQC_REPORT_TIMER_M GENMASK(31, 0) - -/* WORD4 */ -#define HNS_ROCE_EQC_EQE_BA_L_S 0 -#define HNS_ROCE_EQC_EQE_BA_L_M GENMASK(31, 0) - -/* WORD5 */ -#define HNS_ROCE_EQC_EQE_BA_H_S 0 -#define HNS_ROCE_EQC_EQE_BA_H_M GENMASK(28, 0) - -/* WORD6 */ -#define HNS_ROCE_EQC_SHIFT_S 0 -#define HNS_ROCE_EQC_SHIFT_M GENMASK(7, 0) - -#define HNS_ROCE_EQC_MSI_INDX_S 8 -#define HNS_ROCE_EQC_MSI_INDX_M GENMASK(15, 8) - -#define HNS_ROCE_EQC_CUR_EQE_BA_L_S 16 -#define HNS_ROCE_EQC_CUR_EQE_BA_L_M GENMASK(31, 16) - -/* WORD7 */ -#define HNS_ROCE_EQC_CUR_EQE_BA_M_S 0 -#define HNS_ROCE_EQC_CUR_EQE_BA_M_M GENMASK(31, 0) - -/* WORD8 */ -#define HNS_ROCE_EQC_CUR_EQE_BA_H_S 0 -#define HNS_ROCE_EQC_CUR_EQE_BA_H_M GENMASK(3, 0) - -#define HNS_ROCE_EQC_CONS_INDX_S 8 -#define HNS_ROCE_EQC_CONS_INDX_M GENMASK(31, 8) - -/* WORD9 */ -#define HNS_ROCE_EQC_NXT_EQE_BA_L_S 0 -#define HNS_ROCE_EQC_NXT_EQE_BA_L_M GENMASK(31, 0) - -/* WORD10 */ -#define HNS_ROCE_EQC_NXT_EQE_BA_H_S 0 -#define HNS_ROCE_EQC_NXT_EQE_BA_H_M GENMASK(19, 0) - -#define HNS_ROCE_EQC_EQE_SIZE_S 20 -#define HNS_ROCE_EQC_EQE_SIZE_M GENMASK(21, 20) +#define EQC_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_eq_context, h, l) + +#define EQC_EQ_ST EQC_FIELD_LOC(1, 0) +#define EQC_EQE_HOP_NUM EQC_FIELD_LOC(3, 2) +#define EQC_OVER_IGNORE EQC_FIELD_LOC(4, 4) +#define EQC_COALESCE EQC_FIELD_LOC(5, 5) +#define EQC_ARM_ST EQC_FIELD_LOC(7, 6) +#define EQC_EQN EQC_FIELD_LOC(15, 8) +#define EQC_EQE_CNT EQC_FIELD_LOC(31, 16) +#define EQC_EQE_BA_PG_SZ EQC_FIELD_LOC(35, 32) +#define EQC_EQE_BUF_PG_SZ EQC_FIELD_LOC(39, 36) +#define EQC_EQ_PROD_INDX EQC_FIELD_LOC(63, 40) +#define EQC_EQ_MAX_CNT EQC_FIELD_LOC(79, 64) +#define EQC_EQ_PERIOD EQC_FIELD_LOC(95, 80) +#define EQC_EQE_REPORT_TIMER EQC_FIELD_LOC(127, 96) +#define EQC_EQE_BA_L EQC_FIELD_LOC(159, 128) +#define EQC_EQE_BA_H EQC_FIELD_LOC(188, 160) +#define EQC_SHIFT EQC_FIELD_LOC(199, 192) +#define EQC_MSI_INDX EQC_FIELD_LOC(207, 200) +#define EQC_CUR_EQE_BA_L EQC_FIELD_LOC(223, 208) +#define EQC_CUR_EQE_BA_M EQC_FIELD_LOC(255, 224) +#define EQC_CUR_EQE_BA_H EQC_FIELD_LOC(259, 256) +#define EQC_EQ_CONS_INDX EQC_FIELD_LOC(287, 264) +#define EQC_NEX_EQE_BA_L EQC_FIELD_LOC(319, 288) +#define EQC_NEX_EQE_BA_H EQC_FIELD_LOC(339, 320) +#define EQC_EQE_SIZE EQC_FIELD_LOC(341, 340) #define HNS_ROCE_V2_CEQE_COMP_CQN_S 0 #define HNS_ROCE_V2_CEQE_COMP_CQN_M GENMASK(23, 0) @@ -2082,14 +1876,14 @@ struct hns_roce_eq_context { #define HNS_ROCE_V2_AEQE_SUB_TYPE_S 8 #define HNS_ROCE_V2_AEQE_SUB_TYPE_M GENMASK(15, 8) -#define HNS_ROCE_V2_EQ_DB_CMD_S 16 -#define HNS_ROCE_V2_EQ_DB_CMD_M GENMASK(17, 16) +#define V2_EQ_DB_TAG_S 0 +#define V2_EQ_DB_TAG_M GENMASK(7, 0) -#define HNS_ROCE_V2_EQ_DB_TAG_S 0 -#define HNS_ROCE_V2_EQ_DB_TAG_M GENMASK(7, 0) +#define V2_EQ_DB_CMD_S 16 +#define V2_EQ_DB_CMD_M GENMASK(17, 16) -#define HNS_ROCE_V2_EQ_DB_PARA_S 0 -#define HNS_ROCE_V2_EQ_DB_PARA_M GENMASK(23, 0) +#define V2_EQ_DB_CONS_IDX_S 0 +#define V2_EQ_DB_CONS_IDX_M GENMASK(23, 0) #define HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S 0 #define HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M GENMASK(23, 0) diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index c9c0836394a2..6c6e82b11d8b 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -42,7 +42,7 @@ #include "hns_roce_device.h" #include "hns_roce_hem.h" -static int hns_roce_set_mac(struct hns_roce_dev *hr_dev, u8 port, u8 *addr) +static int hns_roce_set_mac(struct hns_roce_dev *hr_dev, u32 port, u8 *addr) { u8 phy_port; u32 i; @@ -63,7 +63,7 @@ static int hns_roce_set_mac(struct hns_roce_dev *hr_dev, u8 port, u8 *addr) static int hns_roce_add_gid(const struct ib_gid_attr *attr, void **context) { struct hns_roce_dev *hr_dev = to_hr_dev(attr->device); - u8 port = attr->port_num - 1; + u32 port = attr->port_num - 1; int ret; if (port >= hr_dev->caps.num_ports) @@ -77,7 +77,7 @@ static int hns_roce_add_gid(const struct ib_gid_attr *attr, void **context) static int hns_roce_del_gid(const struct ib_gid_attr *attr, void **context) { struct hns_roce_dev *hr_dev = to_hr_dev(attr->device); - u8 port = attr->port_num - 1; + u32 port = attr->port_num - 1; int ret; if (port >= hr_dev->caps.num_ports) @@ -88,7 +88,7 @@ static int hns_roce_del_gid(const struct ib_gid_attr *attr, void **context) return ret; } -static int handle_en_event(struct hns_roce_dev *hr_dev, u8 port, +static int handle_en_event(struct hns_roce_dev *hr_dev, u32 port, unsigned long event) { struct device *dev = hr_dev->dev; @@ -128,7 +128,7 @@ static int hns_roce_netdev_event(struct notifier_block *self, struct hns_roce_ib_iboe *iboe = NULL; struct hns_roce_dev *hr_dev = NULL; int ret; - u8 port; + u32 port; hr_dev = container_of(self, struct hns_roce_dev, iboe.nb); iboe = &hr_dev->iboe; @@ -207,10 +207,13 @@ static int hns_roce_query_device(struct ib_device *ib_dev, props->max_fast_reg_page_list_len = HNS_ROCE_FRMR_MAX_PA; } + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC) + props->device_cap_flags |= IB_DEVICE_XRC; + return 0; } -static int hns_roce_query_port(struct ib_device *ib_dev, u8 port_num, +static int hns_roce_query_port(struct ib_device *ib_dev, u32 port_num, struct ib_port_attr *props) { struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); @@ -218,7 +221,7 @@ static int hns_roce_query_port(struct ib_device *ib_dev, u8 port_num, struct net_device *net_dev; unsigned long flags; enum ib_mtu mtu; - u8 port; + u32 port; port = port_num - 1; @@ -258,12 +261,12 @@ static int hns_roce_query_port(struct ib_device *ib_dev, u8 port_num, } static enum rdma_link_layer hns_roce_get_link_layer(struct ib_device *device, - u8 port_num) + u32 port_num) { return IB_LINK_LAYER_ETHERNET; } -static int hns_roce_query_pkey(struct ib_device *ib_dev, u8 port, u16 index, +static int hns_roce_query_pkey(struct ib_device *ib_dev, u32 port, u16 index, u16 *pkey) { *pkey = PKEY_ID; @@ -300,12 +303,14 @@ static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx, return -EAGAIN; resp.qp_tab_size = hr_dev->caps.num_qps; + resp.srq_tab_size = hr_dev->caps.num_srqs; ret = hns_roce_uar_alloc(hr_dev, &context->uar); if (ret) goto error_fail_uar_alloc; - if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) { + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQ_RECORD_DB || + hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) { INIT_LIST_HEAD(&context->page_list); mutex_init(&context->page_mutex); } @@ -365,7 +370,7 @@ static int hns_roce_mmap(struct ib_ucontext *context, } } -static int hns_roce_port_immutable(struct ib_device *ib_dev, u8 port_num, +static int hns_roce_port_immutable(struct ib_device *ib_dev, u32 port_num, struct ib_port_immutable *immutable) { struct ib_port_attr attr; @@ -390,6 +395,19 @@ static void hns_roce_disassociate_ucontext(struct ib_ucontext *ibcontext) { } +static void hns_roce_get_fw_ver(struct ib_device *device, char *str) +{ + u64 fw_ver = to_hr_dev(device)->caps.fw_ver; + unsigned int major, minor, sub_minor; + + major = upper_32_bits(fw_ver); + minor = high_16_bits(lower_32_bits(fw_ver)); + sub_minor = low_16_bits(fw_ver); + + snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u.%04u", major, minor, + sub_minor); +} + static void hns_roce_unregister_device(struct hns_roce_dev *hr_dev) { struct hns_roce_ib_iboe *iboe = &hr_dev->iboe; @@ -405,6 +423,7 @@ static const struct ib_device_ops hns_roce_dev_ops = { .uverbs_abi_ver = 1, .uverbs_no_driver_id_binding = 1, + .get_dev_fw_str = hns_roce_get_fw_ver, .add_gid = hns_roce_add_gid, .alloc_pd = hns_roce_alloc_pd, .alloc_ucontext = hns_roce_alloc_ucontext, @@ -461,6 +480,13 @@ static const struct ib_device_ops hns_roce_dev_srq_ops = { INIT_RDMA_OBJ_SIZE(ib_srq, hns_roce_srq, ibsrq), }; +static const struct ib_device_ops hns_roce_dev_xrcd_ops = { + .alloc_xrcd = hns_roce_alloc_xrcd, + .dealloc_xrcd = hns_roce_dealloc_xrcd, + + INIT_RDMA_OBJ_SIZE(ib_xrcd, hns_roce_xrcd, ibxrcd), +}; + static int hns_roce_register_device(struct hns_roce_dev *hr_dev) { int ret; @@ -484,20 +510,20 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev) if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_REREG_MR) ib_set_device_ops(ib_dev, &hns_roce_dev_mr_ops); - /* MW */ if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_MW) ib_set_device_ops(ib_dev, &hns_roce_dev_mw_ops); - /* FRMR */ if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_FRMR) ib_set_device_ops(ib_dev, &hns_roce_dev_frmr_ops); - /* SRQ */ if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) { ib_set_device_ops(ib_dev, &hns_roce_dev_srq_ops); ib_set_device_ops(ib_dev, hr_dev->hw->hns_roce_dev_srq_ops); } + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC) + ib_set_device_ops(ib_dev, &hns_roce_dev_xrcd_ops); + ib_set_device_ops(ib_dev, hr_dev->hw->hns_roce_dev_ops); ib_set_device_ops(ib_dev, &hns_roce_dev_ops); for (i = 0; i < hr_dev->caps.num_ports; i++) { @@ -704,7 +730,8 @@ static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev) spin_lock_init(&hr_dev->sm_lock); spin_lock_init(&hr_dev->bt_cmd_lock); - if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) { + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQ_RECORD_DB || + hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) { INIT_LIST_HEAD(&hr_dev->pgdir_list); mutex_init(&hr_dev->pgdir_mutex); } @@ -727,10 +754,19 @@ static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev) goto err_uar_alloc_free; } + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC) { + ret = hns_roce_init_xrcd_table(hr_dev); + if (ret) { + dev_err(dev, "failed to init xrcd table, ret = %d.\n", + ret); + goto err_pd_table_free; + } + } + ret = hns_roce_init_mr_table(hr_dev); if (ret) { dev_err(dev, "Failed to init memory region table.\n"); - goto err_pd_table_free; + goto err_xrcd_table_free; } hns_roce_init_cq_table(hr_dev); @@ -759,6 +795,10 @@ err_cq_table_free: hns_roce_cleanup_cq_table(hr_dev); hns_roce_cleanup_mr_table(hr_dev); +err_xrcd_table_free: + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC) + hns_roce_cleanup_xrcd_table(hr_dev); + err_pd_table_free: hns_roce_cleanup_pd_table(hr_dev); @@ -886,6 +926,8 @@ int hns_roce_init(struct hns_roce_dev *hr_dev) INIT_LIST_HEAD(&hr_dev->qp_list); spin_lock_init(&hr_dev->qp_list_lock); + INIT_LIST_HEAD(&hr_dev->dip_list); + spin_lock_init(&hr_dev->dip_list_lock); ret = hns_roce_register_device(hr_dev); if (ret) diff --git a/drivers/infiniband/hw/hns/hns_roce_pd.c b/drivers/infiniband/hw/hns/hns_roce_pd.c index cca818d05a8f..a5813bf567b2 100644 --- a/drivers/infiniband/hw/hns/hns_roce_pd.c +++ b/drivers/infiniband/hw/hns/hns_roce_pd.c @@ -137,3 +137,62 @@ void hns_roce_cleanup_uar_table(struct hns_roce_dev *hr_dev) { hns_roce_bitmap_cleanup(&hr_dev->uar_table.bitmap); } + +static int hns_roce_xrcd_alloc(struct hns_roce_dev *hr_dev, u32 *xrcdn) +{ + unsigned long obj; + int ret; + + ret = hns_roce_bitmap_alloc(&hr_dev->xrcd_bitmap, &obj); + if (ret) + return ret; + + *xrcdn = obj; + + return 0; +} + +static void hns_roce_xrcd_free(struct hns_roce_dev *hr_dev, + u32 xrcdn) +{ + hns_roce_bitmap_free(&hr_dev->xrcd_bitmap, xrcdn, BITMAP_NO_RR); +} + +int hns_roce_init_xrcd_table(struct hns_roce_dev *hr_dev) +{ + return hns_roce_bitmap_init(&hr_dev->xrcd_bitmap, + hr_dev->caps.num_xrcds, + hr_dev->caps.num_xrcds - 1, + hr_dev->caps.reserved_xrcds, 0); +} + +void hns_roce_cleanup_xrcd_table(struct hns_roce_dev *hr_dev) +{ + hns_roce_bitmap_cleanup(&hr_dev->xrcd_bitmap); +} + +int hns_roce_alloc_xrcd(struct ib_xrcd *ib_xrcd, struct ib_udata *udata) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ib_xrcd->device); + struct hns_roce_xrcd *xrcd = to_hr_xrcd(ib_xrcd); + int ret; + + if (!(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC)) + return -EINVAL; + + ret = hns_roce_xrcd_alloc(hr_dev, &xrcd->xrcdn); + if (ret) { + dev_err(hr_dev->dev, "failed to alloc xrcdn, ret = %d.\n", ret); + return ret; + } + + return 0; +} + +int hns_roce_dealloc_xrcd(struct ib_xrcd *ib_xrcd, struct ib_udata *udata) +{ + hns_roce_xrcd_free(to_hr_dev(ib_xrcd->device), + to_hr_xrcd(ib_xrcd)->xrcdn); + + return 0; +} diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index 004aca9086ab..230a909ba9bc 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -98,7 +98,9 @@ void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type) if (hr_dev->hw_rev != HNS_ROCE_HW_VER1 && (event_type == HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR || event_type == HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR || - event_type == HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR)) { + event_type == HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR || + event_type == HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION || + event_type == HNS_ROCE_EVENT_TYPE_INVALID_XRCETH)) { qp->state = IB_QPS_ERR; if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &qp->flush_flag)) init_flush_work(hr_dev, qp); @@ -142,6 +144,8 @@ static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp, event.event = IB_EVENT_QP_REQ_ERR; break; case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: + case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION: + case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH: event.event = IB_EVENT_QP_ACCESS_ERR; break; default: @@ -366,8 +370,13 @@ void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) unsigned long flags; list_del(&hr_qp->node); - list_del(&hr_qp->sq_node); - list_del(&hr_qp->rq_node); + + if (hr_qp->ibqp.qp_type != IB_QPT_XRC_TGT) + list_del(&hr_qp->sq_node); + + if (hr_qp->ibqp.qp_type != IB_QPT_XRC_INI && + hr_qp->ibqp.qp_type != IB_QPT_XRC_TGT) + list_del(&hr_qp->rq_node); xa_lock_irqsave(xa, flags); __xa_erase(xa, hr_qp->qpn & (hr_dev->caps.num_qps - 1)); @@ -478,7 +487,9 @@ static int set_rq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap, hr_qp->rq.max_gs); hr_qp->rq.wqe_cnt = cnt; - if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE && + hr_qp->ibqp.qp_type != IB_QPT_UD && + hr_qp->ibqp.qp_type != IB_QPT_GSI) hr_qp->rq_inl_buf.wqe_cnt = cnt; else hr_qp->rq_inl_buf.wqe_cnt = 0; @@ -776,7 +787,7 @@ static inline bool user_qp_has_sdb(struct hns_roce_dev *hr_dev, struct hns_roce_ib_create_qp_resp *resp, struct hns_roce_ib_create_qp *ucmd) { - return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SQ_RECORD_DB) && + return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) && udata->outlen >= offsetofend(typeof(*resp), cap_flags) && hns_roce_qp_has_sq(init_attr) && udata->inlen >= offsetofend(typeof(*ucmd), sdb_addr)); @@ -787,7 +798,7 @@ static inline bool user_qp_has_rdb(struct hns_roce_dev *hr_dev, struct ib_udata *udata, struct hns_roce_ib_create_qp_resp *resp) { - return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) && + return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) && udata->outlen >= offsetofend(typeof(*resp), cap_flags) && hns_roce_qp_has_rq(init_attr)); } @@ -795,7 +806,7 @@ static inline bool user_qp_has_rdb(struct hns_roce_dev *hr_dev, static inline bool kernel_qp_has_rdb(struct hns_roce_dev *hr_dev, struct ib_qp_init_attr *init_attr) { - return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) && + return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) && hns_roce_qp_has_rq(init_attr)); } @@ -840,11 +851,16 @@ static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, resp->cap_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB; } } else { - /* QP doorbell register address */ - hr_qp->sq.db_reg_l = hr_dev->reg_base + hr_dev->sdb_offset + - DB_REG_OFFSET * hr_dev->priv_uar.index; - hr_qp->rq.db_reg_l = hr_dev->reg_base + hr_dev->odb_offset + - DB_REG_OFFSET * hr_dev->priv_uar.index; + if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) + hr_qp->sq.db_reg = hr_dev->mem_base + + HNS_ROCE_DWQE_SIZE * hr_qp->qpn; + else + hr_qp->sq.db_reg = + hr_dev->reg_base + hr_dev->sdb_offset + + DB_REG_OFFSET * hr_dev->priv_uar.index; + + hr_qp->rq.db_reg = hr_dev->reg_base + hr_dev->odb_offset + + DB_REG_OFFSET * hr_dev->priv_uar.index; if (kernel_qp_has_rdb(hr_dev, init_attr)) { ret = hns_roce_alloc_db(hr_dev, &hr_qp->rdb, 0); @@ -1011,36 +1027,36 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, } } - ret = alloc_qp_db(hr_dev, hr_qp, init_attr, udata, &ucmd, &resp); - if (ret) { - ibdev_err(ibdev, "failed to alloc QP doorbell, ret = %d.\n", - ret); - goto err_wrid; - } - ret = alloc_qp_buf(hr_dev, hr_qp, init_attr, udata, ucmd.buf_addr); if (ret) { ibdev_err(ibdev, "failed to alloc QP buffer, ret = %d.\n", ret); - goto err_db; + goto err_buf; } ret = alloc_qpn(hr_dev, hr_qp); if (ret) { ibdev_err(ibdev, "failed to alloc QPN, ret = %d.\n", ret); - goto err_buf; + goto err_qpn; + } + + ret = alloc_qp_db(hr_dev, hr_qp, init_attr, udata, &ucmd, &resp); + if (ret) { + ibdev_err(ibdev, "failed to alloc QP doorbell, ret = %d.\n", + ret); + goto err_db; } ret = alloc_qpc(hr_dev, hr_qp); if (ret) { ibdev_err(ibdev, "failed to alloc QP context, ret = %d.\n", ret); - goto err_qpn; + goto err_qpc; } ret = hns_roce_qp_store(hr_dev, hr_qp, init_attr); if (ret) { ibdev_err(ibdev, "failed to store QP, ret = %d.\n", ret); - goto err_qpc; + goto err_store; } if (udata) { @@ -1055,7 +1071,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) { ret = hr_dev->hw->qp_flow_control_init(hr_dev, hr_qp); if (ret) - goto err_store; + goto err_flow_ctrl; } hr_qp->ibqp.qp_num = hr_qp->qpn; @@ -1065,17 +1081,17 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, return 0; -err_store: +err_flow_ctrl: hns_roce_qp_remove(hr_dev, hr_qp); -err_qpc: +err_store: free_qpc(hr_dev, hr_qp); -err_qpn: +err_qpc: + free_qp_db(hr_dev, hr_qp, udata); +err_db: free_qpn(hr_dev, hr_qp); -err_buf: +err_qpn: free_qp_buf(hr_dev, hr_qp); -err_db: - free_qp_db(hr_dev, hr_qp, udata); -err_wrid: +err_buf: free_kernel_wrid(hr_qp); return ret; } @@ -1100,11 +1116,16 @@ static int check_qp_type(struct hns_roce_dev *hr_dev, enum ib_qp_type type, bool is_user) { switch (type) { + case IB_QPT_XRC_INI: + case IB_QPT_XRC_TGT: + if (!(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC)) + goto out; + break; case IB_QPT_UD: if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08 && is_user) goto out; - fallthrough; + break; case IB_QPT_RC: case IB_QPT_GSI: break; @@ -1124,8 +1145,8 @@ struct ib_qp *hns_roce_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *init_attr, struct ib_udata *udata) { - struct hns_roce_dev *hr_dev = to_hr_dev(pd->device); - struct ib_device *ibdev = &hr_dev->ib_dev; + struct ib_device *ibdev = pd ? pd->device : init_attr->xrcd->device; + struct hns_roce_dev *hr_dev = to_hr_dev(ibdev); struct hns_roce_qp *hr_qp; int ret; @@ -1137,6 +1158,15 @@ struct ib_qp *hns_roce_create_qp(struct ib_pd *pd, if (!hr_qp) return ERR_PTR(-ENOMEM); + if (init_attr->qp_type == IB_QPT_XRC_INI) + init_attr->recv_cq = NULL; + + if (init_attr->qp_type == IB_QPT_XRC_TGT) { + hr_qp->xrcdn = to_hr_xrcd(init_attr->xrcd)->xrcdn; + init_attr->recv_cq = NULL; + init_attr->send_cq = NULL; + } + if (init_attr->qp_type == IB_QPT_GSI) { hr_qp->port = init_attr->port_num - 1; hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; @@ -1156,20 +1186,18 @@ struct ib_qp *hns_roce_create_qp(struct ib_pd *pd, int to_hr_qp_type(int qp_type) { - int transport_type; - - if (qp_type == IB_QPT_RC) - transport_type = SERV_TYPE_RC; - else if (qp_type == IB_QPT_UC) - transport_type = SERV_TYPE_UC; - else if (qp_type == IB_QPT_UD) - transport_type = SERV_TYPE_UD; - else if (qp_type == IB_QPT_GSI) - transport_type = SERV_TYPE_UD; - else - transport_type = -1; - - return transport_type; + switch (qp_type) { + case IB_QPT_RC: + return SERV_TYPE_RC; + case IB_QPT_UD: + case IB_QPT_GSI: + return SERV_TYPE_UD; + case IB_QPT_XRC_INI: + case IB_QPT_XRC_TGT: + return SERV_TYPE_XRC; + default: + return -1; + } } static int check_mtu_validate(struct hns_roce_dev *hr_dev, diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c index d5a6de0e7095..546d182c577a 100644 --- a/drivers/infiniband/hw/hns/hns_roce_srq.c +++ b/drivers/infiniband/hw/hns/hns_roce_srq.c @@ -314,6 +314,9 @@ static void set_srq_ext_param(struct hns_roce_srq *srq, { srq->cqn = ib_srq_has_cq(init_attr->srq_type) ? to_hr_cq(init_attr->ext.cq)->cqn : 0; + + srq->xrcdn = (init_attr->srq_type == IB_SRQT_XRC) ? + to_hr_xrcd(init_attr->ext.xrc.xrcd)->xrcdn : 0; } static int set_srq_param(struct hns_roce_srq *srq, @@ -412,7 +415,7 @@ int hns_roce_create_srq(struct ib_srq *ib_srq, } } - srq->db_reg_l = hr_dev->reg_base + SRQ_DB_REG; + srq->db_reg = hr_dev->reg_base + SRQ_DB_REG; srq->event = hns_roce_ib_srq_event; atomic_set(&srq->refcount, 1); init_completion(&srq->free); diff --git a/drivers/infiniband/hw/i40iw/i40iw.h b/drivers/infiniband/hw/i40iw/i40iw.h index 6a79502c8b53..be4094ac4fac 100644 --- a/drivers/infiniband/hw/i40iw/i40iw.h +++ b/drivers/infiniband/hw/i40iw/i40iw.h @@ -504,15 +504,6 @@ static inline void i40iw_free_resource(struct i40iw_device *iwdev, spin_unlock_irqrestore(&iwdev->resource_lock, flags); } -/** - * to_iwhdl - Get the handler from the device pointer - * @iwdev: device pointer - **/ -static inline struct i40iw_handler *to_iwhdl(struct i40iw_device *iw_dev) -{ - return container_of(iw_dev, struct i40iw_handler, device); -} - struct i40iw_handler *i40iw_find_netdev(struct net_device *netdev); /** diff --git a/drivers/infiniband/hw/i40iw/i40iw_cm.c b/drivers/infiniband/hw/i40iw/i40iw_cm.c index ac65c8237b2e..2450b7dd51f6 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_cm.c +++ b/drivers/infiniband/hw/i40iw/i40iw_cm.c @@ -905,7 +905,7 @@ static int i40iw_send_mpa_reject(struct i40iw_cm_node *cm_node, } /** - * recv_mpa - process an IETF MPA frame + * i40iw_parse_mpa - process an IETF MPA frame * @cm_node: connection's node * @buffer: Data pointer * @type: to return accept or reject @@ -4360,7 +4360,7 @@ void i40iw_cm_teardown_connections(struct i40iw_device *iwdev, u32 *ipaddr, } /** - * i40iw_ifdown_notify - process an ifdown on an interface + * i40iw_if_notify - process an ifdown on an interface * @iwdev: device pointer * @netdev: network interface device structure * @ipaddr: Pointer to IPv4 or IPv6 address diff --git a/drivers/infiniband/hw/i40iw/i40iw_hmc.c b/drivers/infiniband/hw/i40iw/i40iw_hmc.c index 8bd72af9e099..b44bfc1d239b 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_hmc.c +++ b/drivers/infiniband/hw/i40iw/i40iw_hmc.c @@ -285,7 +285,7 @@ static enum i40iw_status_code i40iw_hmc_finish_add_sd_reg(struct i40iw_sc_dev *d } /** - * i40iw_create_iw_hmc_obj - allocate backing store for hmc objects + * i40iw_sc_create_hmc_obj - allocate backing store for hmc objects * @dev: pointer to the device structure * @info: pointer to i40iw_hmc_iw_create_obj_info struct * @@ -434,7 +434,7 @@ static enum i40iw_status_code i40iw_finish_del_sd_reg(struct i40iw_sc_dev *dev, } /** - * i40iw_del_iw_hmc_obj - remove pe hmc objects + * i40iw_sc_del_hmc_obj - remove pe hmc objects * @dev: pointer to the device structure * @info: pointer to i40iw_hmc_del_obj_info struct * @reset: true if called before reset diff --git a/drivers/infiniband/hw/i40iw/i40iw_main.c b/drivers/infiniband/hw/i40iw/i40iw_main.c index ab4cb11950dc..b496f30ce066 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_main.c +++ b/drivers/infiniband/hw/i40iw/i40iw_main.c @@ -78,7 +78,7 @@ static struct i40e_client i40iw_client; static char i40iw_client_name[I40E_CLIENT_STR_LENGTH] = "i40iw"; static LIST_HEAD(i40iw_handlers); -static spinlock_t i40iw_handler_lock; +static DEFINE_SPINLOCK(i40iw_handler_lock); static enum i40iw_status_code i40iw_virtchnl_send(struct i40iw_sc_dev *dev, u32 vf_id, u8 *msg, u16 len); @@ -251,7 +251,7 @@ static void i40iw_destroy_cqp(struct i40iw_device *iwdev, bool free_hwcqp) } /** - * i40iw_disable_irqs - disable device interrupts + * i40iw_disable_irq - disable device interrupts * @dev: hardware control device structure * @msix_vec: msix vector to disable irq * @dev_id: parameter to pass to free_irq (used during irq setup) @@ -2043,7 +2043,6 @@ static int __init i40iw_init_module(void) i40iw_client.ops = &i40e_ops; memcpy(i40iw_client.name, i40iw_client_name, I40E_CLIENT_STR_LENGTH); i40iw_client.type = I40E_CLIENT_IWARP; - spin_lock_init(&i40iw_handler_lock); ret = i40e_register_client(&i40iw_client); i40iw_register_notifiers(); diff --git a/drivers/infiniband/hw/i40iw/i40iw_osdep.h b/drivers/infiniband/hw/i40iw/i40iw_osdep.h index d474aad62a81..d938ccb195b1 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_osdep.h +++ b/drivers/infiniband/hw/i40iw/i40iw_osdep.h @@ -51,17 +51,6 @@ static inline void set_64bit_val(u64 *wqe_words, u32 byte_index, u64 value) } /** - * set_32bit_val - set 32 value to hw wqe - * @wqe_words: wqe addr to write - * @byte_index: index in wqe - * @value: value to write - **/ -static inline void set_32bit_val(u32 *wqe_words, u32 byte_index, u32 value) -{ - wqe_words[byte_index >> 2] = value; -} - -/** * get_64bit_val - read 64 bit value from wqe * @wqe_words: wqe addr * @byte_index: index to read from @@ -72,17 +61,6 @@ static inline void get_64bit_val(u64 *wqe_words, u32 byte_index, u64 *value) *value = wqe_words[byte_index >> 3]; } -/** - * get_32bit_val - read 32 bit value from wqe - * @wqe_words: wqe addr - * @byte_index: index to reaad from - * @value: return 32 bit value - **/ -static inline void get_32bit_val(u32 *wqe_words, u32 byte_index, u32 *value) -{ - *value = wqe_words[byte_index >> 2]; -} - struct i40iw_dma_mem { void *va; dma_addr_t pa; diff --git a/drivers/infiniband/hw/i40iw/i40iw_pble.c b/drivers/infiniband/hw/i40iw/i40iw_pble.c index 53e5cd1a2bd6..146a4148219b 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_pble.c +++ b/drivers/infiniband/hw/i40iw/i40iw_pble.c @@ -393,12 +393,9 @@ static enum i40iw_status_code add_pble_pool(struct i40iw_sc_dev *dev, i40iw_debug(dev, I40IW_DEBUG_PBLE, "next_fpm_addr = %llx chunk_size[%u] = 0x%x\n", pble_rsrc->next_fpm_addr, chunk->size, chunk->size); pble_rsrc->unallocated_pble -= (chunk->size >> 3); - list_add(&chunk->list, &pble_rsrc->pinfo.clist); sd_reg_val = (sd_entry_type == I40IW_SD_TYPE_PAGED) ? sd_entry->u.pd_table.pd_page_addr.pa : sd_entry->u.bp.addr.pa; - if (sd_entry->valid) - return 0; - if (dev->is_pf) { + if (dev->is_pf && !sd_entry->valid) { ret_code = i40iw_hmc_sd_one(dev, hmc_info->hmc_fn_id, sd_reg_val, idx->sd_idx, sd_entry->entry_type, true); @@ -409,6 +406,7 @@ static enum i40iw_status_code add_pble_pool(struct i40iw_sc_dev *dev, } sd_entry->valid = true; + list_add(&chunk->list, &pble_rsrc->pinfo.clist); return 0; error: kfree(chunk); diff --git a/drivers/infiniband/hw/i40iw/i40iw_puda.c b/drivers/infiniband/hw/i40iw/i40iw_puda.c index d1c8cc0a6236..88fb68e866ba 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_puda.c +++ b/drivers/infiniband/hw/i40iw/i40iw_puda.c @@ -1000,7 +1000,7 @@ static void i40iw_ilq_putback_rcvbuf(struct i40iw_sc_qp *qp, u32 wqe_idx) } /** - * i40iw_ieq_get_fpdu - given length return fpdu length + * i40iw_ieq_get_fpdu_length - given length return fpdu length * @length: length if fpdu */ static u16 i40iw_ieq_get_fpdu_length(u16 length) diff --git a/drivers/infiniband/hw/i40iw/i40iw_utils.c b/drivers/infiniband/hw/i40iw/i40iw_utils.c index 76f052b12c14..9ff825f7860b 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_utils.c +++ b/drivers/infiniband/hw/i40iw/i40iw_utils.c @@ -890,7 +890,7 @@ void i40iw_terminate_done(struct i40iw_sc_qp *qp, int timeout_occurred) } /** - * i40iw_terminate_imeout - timeout happened + * i40iw_terminate_timeout - timeout happened * @t: points to iwarp qp */ static void i40iw_terminate_timeout(struct timer_list *t) diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c index f18d146a6079..b876d722fcc8 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c +++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c @@ -94,7 +94,7 @@ static int i40iw_query_device(struct ib_device *ibdev, * @props: returning device attributes */ static int i40iw_query_port(struct ib_device *ibdev, - u8 port, + u32 port, struct ib_port_attr *props) { props->lid = 1; @@ -647,7 +647,7 @@ error: } /** - * i40iw_query - query qp attributes + * i40iw_query_qp - query qp attributes * @ibqp: qp pointer * @attr: attributes pointer * @attr_mask: Not used @@ -1846,7 +1846,7 @@ static struct ib_mr *i40iw_get_dma_mr(struct ib_pd *pd, int acc) } /** - * i40iw_del_mem_list - Deleting pbl list entries for CQ/QP + * i40iw_del_memlist - Deleting pbl list entries for CQ/QP * @iwmr: iwmr for IB's user page addresses * @ucontext: ptr to user context */ @@ -2347,7 +2347,7 @@ static int i40iw_req_notify_cq(struct ib_cq *ibcq, * @port_num: port number * @immutable: immutable data for the port return */ -static int i40iw_port_immutable(struct ib_device *ibdev, u8 port_num, +static int i40iw_port_immutable(struct ib_device *ibdev, u32 port_num, struct ib_port_immutable *immutable) { struct ib_port_attr attr; @@ -2446,7 +2446,7 @@ static void i40iw_get_dev_fw_str(struct ib_device *dev, char *str) * @port_num: port number */ static struct rdma_hw_stats *i40iw_alloc_hw_stats(struct ib_device *ibdev, - u8 port_num) + u32 port_num) { struct i40iw_device *iwdev = to_iwdev(ibdev); struct i40iw_sc_dev *dev = &iwdev->sc_dev; @@ -2477,7 +2477,7 @@ static struct rdma_hw_stats *i40iw_alloc_hw_stats(struct ib_device *ibdev, */ static int i40iw_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, - u8 port_num, int index) + u32 port_num, int index) { struct i40iw_device *iwdev = to_iwdev(ibdev); struct i40iw_sc_dev *dev = &iwdev->sc_dev; @@ -2504,7 +2504,7 @@ static int i40iw_get_hw_stats(struct ib_device *ibdev, * @gid: Global ID */ static int i40iw_query_gid(struct ib_device *ibdev, - u8 port, + u32 port, int index, union ib_gid *gid) { diff --git a/drivers/infiniband/hw/i40iw/i40iw_virtchnl.c b/drivers/infiniband/hw/i40iw/i40iw_virtchnl.c index aca9061688ae..e34a1522132c 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_virtchnl.c +++ b/drivers/infiniband/hw/i40iw/i40iw_virtchnl.c @@ -333,7 +333,7 @@ static void pf_cqp_get_hmc_fcn_callback(struct i40iw_sc_dev *dev, void *callback } /** - * pf_add_hmc_obj - Callback for Add HMC Object + * pf_add_hmc_obj_callback - Callback for Add HMC Object * @work_vf_dev: pointer to the VF Device */ static void pf_add_hmc_obj_callback(void *work_vf_dev) diff --git a/drivers/infiniband/hw/mlx4/alias_GUID.c b/drivers/infiniband/hw/mlx4/alias_GUID.c index cca414ecfcd5..571d9c542024 100644 --- a/drivers/infiniband/hw/mlx4/alias_GUID.c +++ b/drivers/infiniband/hw/mlx4/alias_GUID.c @@ -73,12 +73,12 @@ static int get_low_record_time_index(struct mlx4_ib_dev *dev, u8 port, int *resched_delay_sec); void mlx4_ib_update_cache_on_guid_change(struct mlx4_ib_dev *dev, int block_num, - u8 port_num, u8 *p_data) + u32 port_num, u8 *p_data) { int i; u64 guid_indexes; int slave_id; - int port_index = port_num - 1; + u32 port_index = port_num - 1; if (!mlx4_is_master(dev->dev)) return; @@ -86,7 +86,7 @@ void mlx4_ib_update_cache_on_guid_change(struct mlx4_ib_dev *dev, int block_num, guid_indexes = be64_to_cpu((__force __be64) dev->sriov.alias_guid. ports_guid[port_num - 1]. all_rec_per_port[block_num].guid_indexes); - pr_debug("port: %d, guid_indexes: 0x%llx\n", port_num, guid_indexes); + pr_debug("port: %u, guid_indexes: 0x%llx\n", port_num, guid_indexes); for (i = 0; i < NUM_ALIAS_GUID_IN_REC; i++) { /* The location of the specific index starts from bit number 4 @@ -184,7 +184,7 @@ unlock: * port_number - 1 or 2 */ void mlx4_ib_notify_slaves_on_guid_change(struct mlx4_ib_dev *dev, - int block_num, u8 port_num, + int block_num, u32 port_num, u8 *p_data) { int i; @@ -206,7 +206,7 @@ void mlx4_ib_notify_slaves_on_guid_change(struct mlx4_ib_dev *dev, guid_indexes = be64_to_cpu((__force __be64) dev->sriov.alias_guid. ports_guid[port_num - 1]. all_rec_per_port[block_num].guid_indexes); - pr_debug("port: %d, guid_indexes: 0x%llx\n", port_num, guid_indexes); + pr_debug("port: %u, guid_indexes: 0x%llx\n", port_num, guid_indexes); /*calculate the slaves and notify them*/ for (i = 0; i < NUM_ALIAS_GUID_IN_REC; i++) { @@ -260,11 +260,11 @@ void mlx4_ib_notify_slaves_on_guid_change(struct mlx4_ib_dev *dev, new_state = set_and_calc_slave_port_state(dev->dev, slave_id, port_num, MLX4_PORT_STATE_IB_PORT_STATE_EVENT_GID_VALID, &gen_event); - pr_debug("slave: %d, port: %d prev_port_state: %d," + pr_debug("slave: %d, port: %u prev_port_state: %d," " new_port_state: %d, gen_event: %d\n", slave_id, port_num, prev_state, new_state, gen_event); if (gen_event == SLAVE_PORT_GEN_EVENT_UP) { - pr_debug("sending PORT_UP event to slave: %d, port: %d\n", + pr_debug("sending PORT_UP event to slave: %d, port: %u\n", slave_id, port_num); mlx4_gen_port_state_change_eqe(dev->dev, slave_id, port_num, MLX4_PORT_CHANGE_SUBTYPE_ACTIVE); @@ -274,7 +274,7 @@ void mlx4_ib_notify_slaves_on_guid_change(struct mlx4_ib_dev *dev, MLX4_PORT_STATE_IB_EVENT_GID_INVALID, &gen_event); if (gen_event == SLAVE_PORT_GEN_EVENT_DOWN) { - pr_debug("sending PORT DOWN event to slave: %d, port: %d\n", + pr_debug("sending PORT DOWN event to slave: %d, port: %u\n", slave_id, port_num); mlx4_gen_port_state_change_eqe(dev->dev, slave_id, diff --git a/drivers/infiniband/hw/mlx4/mad.c b/drivers/infiniband/hw/mlx4/mad.c index f3ace85552f3..d13ecbdd4391 100644 --- a/drivers/infiniband/hw/mlx4/mad.c +++ b/drivers/infiniband/hw/mlx4/mad.c @@ -88,8 +88,8 @@ struct mlx4_rcv_tunnel_mad { struct ib_mad mad; } __packed; -static void handle_client_rereg_event(struct mlx4_ib_dev *dev, u8 port_num); -static void handle_lid_change_event(struct mlx4_ib_dev *dev, u8 port_num); +static void handle_client_rereg_event(struct mlx4_ib_dev *dev, u32 port_num); +static void handle_lid_change_event(struct mlx4_ib_dev *dev, u32 port_num); static void __propagate_pkey_ev(struct mlx4_ib_dev *dev, int port_num, int block, u32 change_bitmap); @@ -186,7 +186,7 @@ int mlx4_MAD_IFC(struct mlx4_ib_dev *dev, int mad_ifc_flags, return err; } -static void update_sm_ah(struct mlx4_ib_dev *dev, u8 port_num, u16 lid, u8 sl) +static void update_sm_ah(struct mlx4_ib_dev *dev, u32 port_num, u16 lid, u8 sl) { struct ib_ah *new_ah; struct rdma_ah_attr ah_attr; @@ -217,8 +217,8 @@ static void update_sm_ah(struct mlx4_ib_dev *dev, u8 port_num, u16 lid, u8 sl) * Snoop SM MADs for port info, GUID info, and P_Key table sets, so we can * synthesize LID change, Client-Rereg, GID change, and P_Key change events. */ -static void smp_snoop(struct ib_device *ibdev, u8 port_num, const struct ib_mad *mad, - u16 prev_lid) +static void smp_snoop(struct ib_device *ibdev, u32 port_num, + const struct ib_mad *mad, u16 prev_lid) { struct ib_port_info *pinfo; u16 lid; @@ -274,7 +274,7 @@ static void smp_snoop(struct ib_device *ibdev, u8 port_num, const struct ib_mad be16_to_cpu(base[i]); } } - pr_debug("PKEY Change event: port=%d, " + pr_debug("PKEY Change event: port=%u, " "block=0x%x, change_bitmap=0x%x\n", port_num, bn, pkey_change_bitmap); @@ -380,7 +380,8 @@ static void node_desc_override(struct ib_device *dev, } } -static void forward_trap(struct mlx4_ib_dev *dev, u8 port_num, const struct ib_mad *mad) +static void forward_trap(struct mlx4_ib_dev *dev, u32 port_num, + const struct ib_mad *mad) { int qpn = mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_SUBN_LID_ROUTED; struct ib_mad_send_buf *send_buf; @@ -429,7 +430,7 @@ static int mlx4_ib_demux_sa_handler(struct ib_device *ibdev, int port, int slave return ret; } -int mlx4_ib_find_real_gid(struct ib_device *ibdev, u8 port, __be64 guid) +int mlx4_ib_find_real_gid(struct ib_device *ibdev, u32 port, __be64 guid) { struct mlx4_ib_dev *dev = to_mdev(ibdev); int i; @@ -443,7 +444,7 @@ int mlx4_ib_find_real_gid(struct ib_device *ibdev, u8 port, __be64 guid) static int find_slave_port_pkey_ix(struct mlx4_ib_dev *dev, int slave, - u8 port, u16 pkey, u16 *ix) + u32 port, u16 pkey, u16 *ix) { int i, ret; u8 unassigned_pkey_ix, pkey_ix, partial_ix = 0xFF; @@ -507,7 +508,7 @@ static int is_proxy_qp0(struct mlx4_ib_dev *dev, int qpn, int slave) return (qpn >= proxy_start && qpn <= proxy_start + 1); } -int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port, +int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u32 port, enum ib_qp_type dest_qpt, struct ib_wc *wc, struct ib_grh *grh, struct ib_mad *mad) { @@ -678,7 +679,7 @@ end: return ret; } -static int mlx4_ib_demux_mad(struct ib_device *ibdev, u8 port, +static int mlx4_ib_demux_mad(struct ib_device *ibdev, u32 port, struct ib_wc *wc, struct ib_grh *grh, struct ib_mad *mad) { @@ -818,7 +819,7 @@ static int mlx4_ib_demux_mad(struct ib_device *ibdev, u8 port, return 0; } -static int ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, +static int ib_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const struct ib_mad *in_mad, struct ib_mad *out_mad) { @@ -932,9 +933,10 @@ static int iboe_process_mad_port_info(void *out_mad) return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; } -static int iboe_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, - const struct ib_wc *in_wc, const struct ib_grh *in_grh, - const struct ib_mad *in_mad, struct ib_mad *out_mad) +static int iboe_process_mad(struct ib_device *ibdev, int mad_flags, + u32 port_num, const struct ib_wc *in_wc, + const struct ib_grh *in_grh, + const struct ib_mad *in_mad, struct ib_mad *out_mad) { struct mlx4_counter counter_stats; struct mlx4_ib_dev *dev = to_mdev(ibdev); @@ -979,7 +981,7 @@ static int iboe_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, return err; } -int mlx4_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, +int mlx4_ib_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const struct ib_mad *in, struct ib_mad *out, size_t *out_mad_size, u16 *out_mad_pkey_index) @@ -1073,7 +1075,7 @@ void mlx4_ib_mad_cleanup(struct mlx4_ib_dev *dev) } } -static void handle_lid_change_event(struct mlx4_ib_dev *dev, u8 port_num) +static void handle_lid_change_event(struct mlx4_ib_dev *dev, u32 port_num) { mlx4_ib_dispatch_event(dev, port_num, IB_EVENT_LID_CHANGE); @@ -1082,7 +1084,7 @@ static void handle_lid_change_event(struct mlx4_ib_dev *dev, u8 port_num) MLX4_EQ_PORT_INFO_LID_CHANGE_MASK); } -static void handle_client_rereg_event(struct mlx4_ib_dev *dev, u8 port_num) +static void handle_client_rereg_event(struct mlx4_ib_dev *dev, u32 port_num) { /* re-configure the alias-guid and mcg's */ if (mlx4_is_master(dev->dev)) { @@ -1121,7 +1123,7 @@ static void propagate_pkey_ev(struct mlx4_ib_dev *dev, int port_num, GET_MASK_FROM_EQE(eqe)); } -static void handle_slaves_guid_change(struct mlx4_ib_dev *dev, u8 port_num, +static void handle_slaves_guid_change(struct mlx4_ib_dev *dev, u32 port_num, u32 guid_tbl_blk_num, u32 change_bitmap) { struct ib_smp *in_mad = NULL; @@ -1177,7 +1179,7 @@ void handle_port_mgmt_change_event(struct work_struct *work) struct ib_event_work *ew = container_of(work, struct ib_event_work, work); struct mlx4_ib_dev *dev = ew->ib_dev; struct mlx4_eqe *eqe = &(ew->ib_eqe); - u8 port = eqe->event.port_mgmt_change.port; + u32 port = eqe->event.port_mgmt_change.port; u32 changed_attr; u32 tbl_block; u32 change_bitmap; @@ -1274,7 +1276,7 @@ void handle_port_mgmt_change_event(struct work_struct *work) kfree(ew); } -void mlx4_ib_dispatch_event(struct mlx4_ib_dev *dev, u8 port_num, +void mlx4_ib_dispatch_event(struct mlx4_ib_dev *dev, u32 port_num, enum ib_event_type type) { struct ib_event event; @@ -1351,7 +1353,7 @@ static int mlx4_ib_multiplex_sa_handler(struct ib_device *ibdev, int port, return ret; } -int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port, +int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u32 port, enum ib_qp_type dest_qpt, u16 pkey_index, u32 remote_qpn, u32 qkey, struct rdma_ah_attr *attr, u8 *s_mac, u16 vlan_id, struct ib_mad *mad) diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c index f26a0d920842..22898d97ecbd 100644 --- a/drivers/infiniband/hw/mlx4/main.c +++ b/drivers/infiniband/hw/mlx4/main.c @@ -81,7 +81,7 @@ static const char mlx4_ib_version[] = static void do_slave_init(struct mlx4_ib_dev *ibdev, int slave, int do_init); static enum rdma_link_layer mlx4_ib_port_link_layer(struct ib_device *device, - u8 port_num); + u32 port_num); static struct workqueue_struct *wq; @@ -129,7 +129,8 @@ static int num_ib_ports(struct mlx4_dev *dev) return ib_ports; } -static struct net_device *mlx4_ib_get_netdev(struct ib_device *device, u8 port_num) +static struct net_device *mlx4_ib_get_netdev(struct ib_device *device, + u32 port_num) { struct mlx4_ib_dev *ibdev = to_mdev(device); struct net_device *dev; @@ -160,7 +161,7 @@ static struct net_device *mlx4_ib_get_netdev(struct ib_device *device, u8 port_n static int mlx4_ib_update_gids_v1(struct gid_entry *gids, struct mlx4_ib_dev *ibdev, - u8 port_num) + u32 port_num) { struct mlx4_cmd_mailbox *mailbox; int err; @@ -193,7 +194,7 @@ static int mlx4_ib_update_gids_v1(struct gid_entry *gids, static int mlx4_ib_update_gids_v1_v2(struct gid_entry *gids, struct mlx4_ib_dev *ibdev, - u8 port_num) + u32 port_num) { struct mlx4_cmd_mailbox *mailbox; int err; @@ -238,7 +239,7 @@ static int mlx4_ib_update_gids_v1_v2(struct gid_entry *gids, static int mlx4_ib_update_gids(struct gid_entry *gids, struct mlx4_ib_dev *ibdev, - u8 port_num) + u32 port_num) { if (ibdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ROCE_V1_V2) return mlx4_ib_update_gids_v1_v2(gids, ibdev, port_num); @@ -407,7 +408,7 @@ int mlx4_ib_gid_index_to_real_index(struct mlx4_ib_dev *ibdev, int real_index = -EINVAL; int i; unsigned long flags; - u8 port_num = attr->port_num; + u32 port_num = attr->port_num; if (port_num > MLX4_MAX_PORTS) return -EINVAL; @@ -649,7 +650,7 @@ out: } static enum rdma_link_layer -mlx4_ib_port_link_layer(struct ib_device *device, u8 port_num) +mlx4_ib_port_link_layer(struct ib_device *device, u32 port_num) { struct mlx4_dev *dev = to_mdev(device)->dev; @@ -657,7 +658,7 @@ mlx4_ib_port_link_layer(struct ib_device *device, u8 port_num) IB_LINK_LAYER_INFINIBAND : IB_LINK_LAYER_ETHERNET; } -static int ib_link_query_port(struct ib_device *ibdev, u8 port, +static int ib_link_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props, int netw_view) { struct ib_smp *in_mad = NULL; @@ -753,7 +754,7 @@ static u8 state_to_phys_state(enum ib_port_state state) IB_PORT_PHYS_STATE_LINK_UP : IB_PORT_PHYS_STATE_DISABLED; } -static int eth_link_query_port(struct ib_device *ibdev, u8 port, +static int eth_link_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props) { @@ -814,7 +815,7 @@ out: return err; } -int __mlx4_ib_query_port(struct ib_device *ibdev, u8 port, +int __mlx4_ib_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props, int netw_view) { int err; @@ -828,14 +829,14 @@ int __mlx4_ib_query_port(struct ib_device *ibdev, u8 port, return err; } -static int mlx4_ib_query_port(struct ib_device *ibdev, u8 port, +static int mlx4_ib_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props) { /* returns host view */ return __mlx4_ib_query_port(ibdev, port, props, 0); } -int __mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index, +int __mlx4_ib_query_gid(struct ib_device *ibdev, u32 port, int index, union ib_gid *gid, int netw_view) { struct ib_smp *in_mad = NULL; @@ -891,7 +892,7 @@ out: return err; } -static int mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index, +static int mlx4_ib_query_gid(struct ib_device *ibdev, u32 port, int index, union ib_gid *gid) { if (rdma_protocol_ib(ibdev, port)) @@ -899,7 +900,8 @@ static int mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index, return 0; } -static int mlx4_ib_query_sl2vl(struct ib_device *ibdev, u8 port, u64 *sl2vl_tbl) +static int mlx4_ib_query_sl2vl(struct ib_device *ibdev, u32 port, + u64 *sl2vl_tbl) { union sl2vl_tbl_to_u64 sl2vl64; struct ib_smp *in_mad = NULL; @@ -959,7 +961,7 @@ static void mlx4_init_sl2vl_tbl(struct mlx4_ib_dev *mdev) } } -int __mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index, +int __mlx4_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey, int netw_view) { struct ib_smp *in_mad = NULL; @@ -992,7 +994,8 @@ out: return err; } -static int mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) +static int mlx4_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index, + u16 *pkey) { return __mlx4_ib_query_pkey(ibdev, port, index, pkey, 0); } @@ -1033,8 +1036,8 @@ static int mlx4_ib_modify_device(struct ib_device *ibdev, int mask, return 0; } -static int mlx4_ib_SET_PORT(struct mlx4_ib_dev *dev, u8 port, int reset_qkey_viols, - u32 cap_mask) +static int mlx4_ib_SET_PORT(struct mlx4_ib_dev *dev, u32 port, + int reset_qkey_viols, u32 cap_mask) { struct mlx4_cmd_mailbox *mailbox; int err; @@ -1059,7 +1062,7 @@ static int mlx4_ib_SET_PORT(struct mlx4_ib_dev *dev, u8 port, int reset_qkey_vio return err; } -static int mlx4_ib_modify_port(struct ib_device *ibdev, u8 port, int mask, +static int mlx4_ib_modify_port(struct ib_device *ibdev, u32 port, int mask, struct ib_port_modify *props) { struct mlx4_ib_dev *mdev = to_mdev(ibdev); @@ -2103,7 +2106,7 @@ static const struct diag_counter diag_device_only[] = { }; static struct rdma_hw_stats *mlx4_ib_alloc_hw_stats(struct ib_device *ibdev, - u8 port_num) + u32 port_num) { struct mlx4_ib_dev *dev = to_mdev(ibdev); struct mlx4_ib_diag_counters *diag = dev->diag_counters; @@ -2118,7 +2121,7 @@ static struct rdma_hw_stats *mlx4_ib_alloc_hw_stats(struct ib_device *ibdev, static int mlx4_ib_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, - u8 port, int index) + u32 port, int index) { struct mlx4_ib_dev *dev = to_mdev(ibdev); struct mlx4_ib_diag_counters *diag = dev->diag_counters; @@ -2466,7 +2469,7 @@ static void mlx4_ib_free_eqs(struct mlx4_dev *dev, struct mlx4_ib_dev *ibdev) ibdev->eq_table = NULL; } -static int mlx4_port_immutable(struct ib_device *ibdev, u8 port_num, +static int mlx4_port_immutable(struct ib_device *ibdev, u32 port_num, struct ib_port_immutable *immutable) { struct ib_port_attr attr; diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h index 78c9bb79ec75..e856cf23a0a1 100644 --- a/drivers/infiniband/hw/mlx4/mlx4_ib.h +++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h @@ -429,7 +429,7 @@ struct mlx4_sriov_alias_guid_port_rec_det { struct mlx4_sriov_alias_guid_info_rec_det all_rec_per_port[NUM_ALIAS_GUID_REC_IN_PORT]; struct workqueue_struct *wq; struct delayed_work alias_guid_work; - u8 port; + u32 port; u32 state_flags; struct mlx4_sriov_alias_guid *parent; struct list_head cb_list; @@ -657,7 +657,7 @@ struct mlx4_ib_qp_tunnel_init_attr { struct ib_qp_init_attr init_attr; int slave; enum ib_qp_type proxy_qp_type; - u8 port; + u32 port; }; struct mlx4_uverbs_ex_query_device { @@ -810,24 +810,24 @@ int mlx4_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, int mlx4_MAD_IFC(struct mlx4_ib_dev *dev, int mad_ifc_flags, int port, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const void *in_mad, void *response_mad); -int mlx4_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, +int mlx4_ib_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const struct ib_mad *in, struct ib_mad *out, size_t *out_mad_size, u16 *out_mad_pkey_index); int mlx4_ib_mad_init(struct mlx4_ib_dev *dev); void mlx4_ib_mad_cleanup(struct mlx4_ib_dev *dev); -int __mlx4_ib_query_port(struct ib_device *ibdev, u8 port, +int __mlx4_ib_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props, int netw_view); -int __mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index, +int __mlx4_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey, int netw_view); -int __mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index, +int __mlx4_ib_query_gid(struct ib_device *ibdev, u32 port, int index, union ib_gid *gid, int netw_view); static inline bool mlx4_ib_ah_grh_present(struct mlx4_ib_ah *ah) { - u8 port = be32_to_cpu(ah->av.ib.port_pd) >> 24 & 3; + u32 port = be32_to_cpu(ah->av.ib.port_pd) >> 24 & 3; if (rdma_port_get_link_layer(ah->ibah.device, port) == IB_LINK_LAYER_ETHERNET) return true; @@ -841,7 +841,7 @@ void clean_vf_mcast(struct mlx4_ib_demux_ctx *ctx, int slave); int mlx4_ib_mcg_init(void); void mlx4_ib_mcg_destroy(void); -int mlx4_ib_find_real_gid(struct ib_device *ibdev, u8 port, __be64 guid); +int mlx4_ib_find_real_gid(struct ib_device *ibdev, u32 port, __be64 guid); int mlx4_ib_mcg_multiplex_handler(struct ib_device *ibdev, int port, int slave, struct ib_sa_mad *sa_mad); @@ -851,16 +851,16 @@ int mlx4_ib_mcg_demux_handler(struct ib_device *ibdev, int port, int slave, int mlx4_ib_add_mc(struct mlx4_ib_dev *mdev, struct mlx4_ib_qp *mqp, union ib_gid *gid); -void mlx4_ib_dispatch_event(struct mlx4_ib_dev *dev, u8 port_num, +void mlx4_ib_dispatch_event(struct mlx4_ib_dev *dev, u32 port_num, enum ib_event_type type); void mlx4_ib_tunnels_update_work(struct work_struct *work); -int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port, +int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u32 port, enum ib_qp_type qpt, struct ib_wc *wc, struct ib_grh *grh, struct ib_mad *mad); -int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port, +int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u32 port, enum ib_qp_type dest_qpt, u16 pkey_index, u32 remote_qpn, u32 qkey, struct rdma_ah_attr *attr, u8 *s_mac, u16 vlan_id, struct ib_mad *mad); @@ -884,10 +884,10 @@ void mlx4_ib_invalidate_all_guid_record(struct mlx4_ib_dev *dev, int port); void mlx4_ib_notify_slaves_on_guid_change(struct mlx4_ib_dev *dev, int block_num, - u8 port_num, u8 *p_data); + u32 port_num, u8 *p_data); void mlx4_ib_update_cache_on_guid_change(struct mlx4_ib_dev *dev, - int block_num, u8 port_num, + int block_num, u32 port_num, u8 *p_data); int add_sysfs_port_mcg_attr(struct mlx4_ib_dev *device, int port_num, diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c index 651785bd57f2..92ddbcc00eb2 100644 --- a/drivers/infiniband/hw/mlx4/qp.c +++ b/drivers/infiniband/hw/mlx4/qp.c @@ -3135,7 +3135,6 @@ static int build_mlx_header(struct mlx4_ib_qp *qp, const struct ib_ud_wr *wr, } if (is_eth) { - struct in6_addr in6; u16 ether_type; u16 pcp = (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 29) << 13; @@ -3148,8 +3147,6 @@ static int build_mlx_header(struct mlx4_ib_qp *qp, const struct ib_ud_wr *wr, memcpy(sqp->ud_header.eth.dmac_h, ah->av.eth.mac, 6); memcpy(&ctrl->srcrb_flags16[0], ah->av.eth.mac, 2); memcpy(&ctrl->imm, ah->av.eth.mac + 2, 4); - memcpy(&in6, sgid.raw, sizeof(in6)); - if (!memcmp(sqp->ud_header.eth.smac_h, sqp->ud_header.eth.dmac_h, 6)) mlx->flags |= cpu_to_be32(MLX4_WQE_CTRL_FORCE_LOOPBACK); diff --git a/drivers/infiniband/hw/mlx5/Makefile b/drivers/infiniband/hw/mlx5/Makefile index b4c009bb0db6..f43380106bd0 100644 --- a/drivers/infiniband/hw/mlx5/Makefile +++ b/drivers/infiniband/hw/mlx5/Makefile @@ -6,6 +6,7 @@ mlx5_ib-y := ah.o \ cong.o \ counters.o \ cq.o \ + dm.o \ doorbell.o \ gsi.o \ ib_virt.o \ diff --git a/drivers/infiniband/hw/mlx5/cmd.c b/drivers/infiniband/hw/mlx5/cmd.c index 234f29912ba9..a8db8a051170 100644 --- a/drivers/infiniband/hw/mlx5/cmd.c +++ b/drivers/infiniband/hw/mlx5/cmd.c @@ -47,107 +47,6 @@ int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point, return mlx5_cmd_exec_inout(dev, query_cong_params, in, out); } -int mlx5_cmd_alloc_memic(struct mlx5_dm *dm, phys_addr_t *addr, - u64 length, u32 alignment) -{ - struct mlx5_core_dev *dev = dm->dev; - u64 num_memic_hw_pages = MLX5_CAP_DEV_MEM(dev, memic_bar_size) - >> PAGE_SHIFT; - u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr); - u32 max_alignment = MLX5_CAP_DEV_MEM(dev, log_max_memic_addr_alignment); - u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE); - u32 out[MLX5_ST_SZ_DW(alloc_memic_out)] = {}; - u32 in[MLX5_ST_SZ_DW(alloc_memic_in)] = {}; - u32 mlx5_alignment; - u64 page_idx = 0; - int ret = 0; - - if (!length || (length & MLX5_MEMIC_ALLOC_SIZE_MASK)) - return -EINVAL; - - /* mlx5 device sets alignment as 64*2^driver_value - * so normalizing is needed. - */ - mlx5_alignment = (alignment < MLX5_MEMIC_BASE_ALIGN) ? 0 : - alignment - MLX5_MEMIC_BASE_ALIGN; - if (mlx5_alignment > max_alignment) - return -EINVAL; - - MLX5_SET(alloc_memic_in, in, opcode, MLX5_CMD_OP_ALLOC_MEMIC); - MLX5_SET(alloc_memic_in, in, range_size, num_pages * PAGE_SIZE); - MLX5_SET(alloc_memic_in, in, memic_size, length); - MLX5_SET(alloc_memic_in, in, log_memic_addr_alignment, - mlx5_alignment); - - while (page_idx < num_memic_hw_pages) { - spin_lock(&dm->lock); - page_idx = bitmap_find_next_zero_area(dm->memic_alloc_pages, - num_memic_hw_pages, - page_idx, - num_pages, 0); - - if (page_idx < num_memic_hw_pages) - bitmap_set(dm->memic_alloc_pages, - page_idx, num_pages); - - spin_unlock(&dm->lock); - - if (page_idx >= num_memic_hw_pages) - break; - - MLX5_SET64(alloc_memic_in, in, range_start_addr, - hw_start_addr + (page_idx * PAGE_SIZE)); - - ret = mlx5_cmd_exec_inout(dev, alloc_memic, in, out); - if (ret) { - spin_lock(&dm->lock); - bitmap_clear(dm->memic_alloc_pages, - page_idx, num_pages); - spin_unlock(&dm->lock); - - if (ret == -EAGAIN) { - page_idx++; - continue; - } - - return ret; - } - - *addr = dev->bar_addr + - MLX5_GET64(alloc_memic_out, out, memic_start_addr); - - return 0; - } - - return -ENOMEM; -} - -void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, u64 length) -{ - struct mlx5_core_dev *dev = dm->dev; - u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr); - u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE); - u32 in[MLX5_ST_SZ_DW(dealloc_memic_in)] = {}; - u64 start_page_idx; - int err; - - addr -= dev->bar_addr; - start_page_idx = (addr - hw_start_addr) >> PAGE_SHIFT; - - MLX5_SET(dealloc_memic_in, in, opcode, MLX5_CMD_OP_DEALLOC_MEMIC); - MLX5_SET64(dealloc_memic_in, in, memic_start_addr, addr); - MLX5_SET(dealloc_memic_in, in, memic_size, length); - - err = mlx5_cmd_exec_in(dev, dealloc_memic, in); - if (err) - return; - - spin_lock(&dm->lock); - bitmap_clear(dm->memic_alloc_pages, - start_page_idx, num_pages); - spin_unlock(&dm->lock); -} - void mlx5_cmd_destroy_tir(struct mlx5_core_dev *dev, u32 tirn, u16 uid) { u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {}; diff --git a/drivers/infiniband/hw/mlx5/cmd.h b/drivers/infiniband/hw/mlx5/cmd.h index 88ea6ef8f2cb..66c96292ed43 100644 --- a/drivers/infiniband/hw/mlx5/cmd.h +++ b/drivers/infiniband/hw/mlx5/cmd.h @@ -41,9 +41,6 @@ int mlx5_cmd_dump_fill_mkey(struct mlx5_core_dev *dev, u32 *mkey); int mlx5_cmd_null_mkey(struct mlx5_core_dev *dev, u32 *null_mkey); int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point, void *out); -int mlx5_cmd_alloc_memic(struct mlx5_dm *dm, phys_addr_t *addr, - u64 length, u32 alignment); -void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, u64 length); int mlx5_cmd_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn, u16 uid); void mlx5_cmd_destroy_tir(struct mlx5_core_dev *dev, u32 tirn, u16 uid); void mlx5_cmd_destroy_tis(struct mlx5_core_dev *dev, u32 tisn, u16 uid); diff --git a/drivers/infiniband/hw/mlx5/cong.c b/drivers/infiniband/hw/mlx5/cong.c index b9291e482428..0b61df52332a 100644 --- a/drivers/infiniband/hw/mlx5/cong.c +++ b/drivers/infiniband/hw/mlx5/cong.c @@ -267,7 +267,7 @@ static void mlx5_ib_set_cc_param_mask_val(void *field, int offset, } } -static int mlx5_ib_get_cc_params(struct mlx5_ib_dev *dev, u8 port_num, +static int mlx5_ib_get_cc_params(struct mlx5_ib_dev *dev, u32 port_num, int offset, u32 *var) { int outlen = MLX5_ST_SZ_BYTES(query_cong_params_out); @@ -304,7 +304,7 @@ alloc_err: return err; } -static int mlx5_ib_set_cc_params(struct mlx5_ib_dev *dev, u8 port_num, +static int mlx5_ib_set_cc_params(struct mlx5_ib_dev *dev, u32 port_num, int offset, u32 var) { int inlen = MLX5_ST_SZ_BYTES(modify_cong_params_in); @@ -397,7 +397,7 @@ static const struct file_operations dbg_cc_fops = { .read = get_param, }; -void mlx5_ib_cleanup_cong_debugfs(struct mlx5_ib_dev *dev, u8 port_num) +void mlx5_ib_cleanup_cong_debugfs(struct mlx5_ib_dev *dev, u32 port_num) { if (!mlx5_debugfs_root || !dev->port[port_num].dbg_cc_params || @@ -409,7 +409,7 @@ void mlx5_ib_cleanup_cong_debugfs(struct mlx5_ib_dev *dev, u8 port_num) dev->port[port_num].dbg_cc_params = NULL; } -void mlx5_ib_init_cong_debugfs(struct mlx5_ib_dev *dev, u8 port_num) +void mlx5_ib_init_cong_debugfs(struct mlx5_ib_dev *dev, u32 port_num) { struct mlx5_ib_dbg_cc_params *dbg_cc_params; struct mlx5_core_dev *mdev; diff --git a/drivers/infiniband/hw/mlx5/counters.c b/drivers/infiniband/hw/mlx5/counters.c index 084652e2b15a..e365341057cb 100644 --- a/drivers/infiniband/hw/mlx5/counters.c +++ b/drivers/infiniband/hw/mlx5/counters.c @@ -139,7 +139,7 @@ static int mlx5_ib_create_counters(struct ib_counters *counters, static const struct mlx5_ib_counters *get_counters(struct mlx5_ib_dev *dev, - u8 port_num) + u32 port_num) { return is_mdev_switchdev_mode(dev->mdev) ? &dev->port[0].cnts : &dev->port[port_num].cnts; @@ -154,7 +154,7 @@ static const struct mlx5_ib_counters *get_counters(struct mlx5_ib_dev *dev, * device port combination in switchdev and non switchdev mode of the * parent device. */ -u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u8 port_num) +u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u32 port_num) { const struct mlx5_ib_counters *cnts = get_counters(dev, port_num); @@ -162,7 +162,7 @@ u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u8 port_num) } static struct rdma_hw_stats *mlx5_ib_alloc_hw_stats(struct ib_device *ibdev, - u8 port_num) + u32 port_num) { struct mlx5_ib_dev *dev = to_mdev(ibdev); const struct mlx5_ib_counters *cnts; @@ -236,13 +236,13 @@ free: static int mlx5_ib_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, - u8 port_num, int index) + u32 port_num, int index) { struct mlx5_ib_dev *dev = to_mdev(ibdev); const struct mlx5_ib_counters *cnts = get_counters(dev, port_num - 1); struct mlx5_core_dev *mdev; int ret, num_counters; - u8 mdev_port_num; + u32 mdev_port_num; if (!stats) return -EINVAL; diff --git a/drivers/infiniband/hw/mlx5/counters.h b/drivers/infiniband/hw/mlx5/counters.h index 1aa30c2f3f4d..6bcaaa52e2b2 100644 --- a/drivers/infiniband/hw/mlx5/counters.h +++ b/drivers/infiniband/hw/mlx5/counters.h @@ -13,5 +13,5 @@ void mlx5_ib_counters_cleanup(struct mlx5_ib_dev *dev); void mlx5_ib_counters_clear_description(struct ib_counters *counters); int mlx5_ib_flow_counters_set_data(struct ib_counters *ibcounters, struct mlx5_ib_create_flow *ucmd); -u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u8 port_num); +u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u32 port_num); #endif /* _MLX5_IB_COUNTERS_H */ diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c index 07b8350929cd..a0b677accd96 100644 --- a/drivers/infiniband/hw/mlx5/devx.c +++ b/drivers/infiniband/hw/mlx5/devx.c @@ -2185,27 +2185,69 @@ static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext, return 0; } +static unsigned int devx_umem_find_best_pgsize(struct ib_umem *umem, + unsigned long pgsz_bitmap) +{ + unsigned long page_size; + + /* Don't bother checking larger page sizes as offset must be zero and + * total DEVX umem length must be equal to total umem length. + */ + pgsz_bitmap &= GENMASK_ULL(max_t(u64, order_base_2(umem->length), + PAGE_SHIFT), + MLX5_ADAPTER_PAGE_SHIFT); + if (!pgsz_bitmap) + return 0; + + page_size = ib_umem_find_best_pgoff(umem, pgsz_bitmap, U64_MAX); + if (!page_size) + return 0; + + /* If the page_size is less than the CPU page size then we can use the + * offset and create a umem which is a subset of the page list. + * For larger page sizes we can't be sure the DMA list reflects the + * VA so we must ensure that the umem extent is exactly equal to the + * page list. Reduce the page size until one of these cases is true. + */ + while ((ib_umem_dma_offset(umem, page_size) != 0 || + (umem->length % page_size) != 0) && + page_size > PAGE_SIZE) + page_size /= 2; + + return page_size; +} + static int devx_umem_reg_cmd_alloc(struct mlx5_ib_dev *dev, struct uverbs_attr_bundle *attrs, struct devx_umem *obj, struct devx_umem_reg_cmd *cmd) { + unsigned long pgsz_bitmap; unsigned int page_size; __be64 *mtt; void *umem; + int ret; /* - * We don't know what the user intends to use this umem for, but the HW - * restrictions must be met. MR, doorbell records, QP, WQ and CQ all - * have different requirements. Since we have no idea how to sort this - * out, only support PAGE_SIZE with the expectation that userspace will - * provide the necessary alignments inside the known PAGE_SIZE and that - * FW will check everything. + * If the user does not pass in pgsz_bitmap then the user promises not + * to use umem_offset!=0 in any commands that allocate on top of the + * umem. + * + * If the user wants to use a umem_offset then it must pass in + * pgsz_bitmap which guides the maximum page size and thus maximum + * object alignment inside the umem. See the PRM. + * + * Users are not allowed to use IOVA here, mkeys are not supported on + * umem. */ - page_size = ib_umem_find_best_pgoff( - obj->umem, PAGE_SIZE, - __mlx5_page_offset_to_bitmask(__mlx5_bit_sz(umem, page_offset), - 0)); + ret = uverbs_get_const_default(&pgsz_bitmap, attrs, + MLX5_IB_ATTR_DEVX_UMEM_REG_PGSZ_BITMAP, + GENMASK_ULL(63, + min(PAGE_SHIFT, MLX5_ADAPTER_PAGE_SHIFT))); + if (ret) + return ret; + + page_size = devx_umem_find_best_pgsize(obj->umem, pgsz_bitmap); if (!page_size) return -EINVAL; @@ -2791,6 +2833,8 @@ DECLARE_UVERBS_NAMED_METHOD( UA_MANDATORY), UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS, enum ib_access_flags), + UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_PGSZ_BITMAP, + u64), UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, UVERBS_ATTR_TYPE(u32), UA_MANDATORY)); diff --git a/drivers/infiniband/hw/mlx5/dm.c b/drivers/infiniband/hw/mlx5/dm.c new file mode 100644 index 000000000000..094bf85589db --- /dev/null +++ b/drivers/infiniband/hw/mlx5/dm.c @@ -0,0 +1,587 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* + * Copyright (c) 2021, Mellanox Technologies inc. All rights reserved. + */ + +#include <rdma/uverbs_std_types.h> +#include "dm.h" + +#define UVERBS_MODULE_NAME mlx5_ib +#include <rdma/uverbs_named_ioctl.h> + +static int mlx5_cmd_alloc_memic(struct mlx5_dm *dm, phys_addr_t *addr, + u64 length, u32 alignment) +{ + struct mlx5_core_dev *dev = dm->dev; + u64 num_memic_hw_pages = MLX5_CAP_DEV_MEM(dev, memic_bar_size) + >> PAGE_SHIFT; + u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr); + u32 max_alignment = MLX5_CAP_DEV_MEM(dev, log_max_memic_addr_alignment); + u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE); + u32 out[MLX5_ST_SZ_DW(alloc_memic_out)] = {}; + u32 in[MLX5_ST_SZ_DW(alloc_memic_in)] = {}; + u32 mlx5_alignment; + u64 page_idx = 0; + int ret = 0; + + if (!length || (length & MLX5_MEMIC_ALLOC_SIZE_MASK)) + return -EINVAL; + + /* mlx5 device sets alignment as 64*2^driver_value + * so normalizing is needed. + */ + mlx5_alignment = (alignment < MLX5_MEMIC_BASE_ALIGN) ? 0 : + alignment - MLX5_MEMIC_BASE_ALIGN; + if (mlx5_alignment > max_alignment) + return -EINVAL; + + MLX5_SET(alloc_memic_in, in, opcode, MLX5_CMD_OP_ALLOC_MEMIC); + MLX5_SET(alloc_memic_in, in, range_size, num_pages * PAGE_SIZE); + MLX5_SET(alloc_memic_in, in, memic_size, length); + MLX5_SET(alloc_memic_in, in, log_memic_addr_alignment, + mlx5_alignment); + + while (page_idx < num_memic_hw_pages) { + spin_lock(&dm->lock); + page_idx = bitmap_find_next_zero_area(dm->memic_alloc_pages, + num_memic_hw_pages, + page_idx, + num_pages, 0); + + if (page_idx < num_memic_hw_pages) + bitmap_set(dm->memic_alloc_pages, + page_idx, num_pages); + + spin_unlock(&dm->lock); + + if (page_idx >= num_memic_hw_pages) + break; + + MLX5_SET64(alloc_memic_in, in, range_start_addr, + hw_start_addr + (page_idx * PAGE_SIZE)); + + ret = mlx5_cmd_exec_inout(dev, alloc_memic, in, out); + if (ret) { + spin_lock(&dm->lock); + bitmap_clear(dm->memic_alloc_pages, + page_idx, num_pages); + spin_unlock(&dm->lock); + + if (ret == -EAGAIN) { + page_idx++; + continue; + } + + return ret; + } + + *addr = dev->bar_addr + + MLX5_GET64(alloc_memic_out, out, memic_start_addr); + + return 0; + } + + return -ENOMEM; +} + +void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, + u64 length) +{ + struct mlx5_core_dev *dev = dm->dev; + u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr); + u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE); + u32 in[MLX5_ST_SZ_DW(dealloc_memic_in)] = {}; + u64 start_page_idx; + int err; + + addr -= dev->bar_addr; + start_page_idx = (addr - hw_start_addr) >> PAGE_SHIFT; + + MLX5_SET(dealloc_memic_in, in, opcode, MLX5_CMD_OP_DEALLOC_MEMIC); + MLX5_SET64(dealloc_memic_in, in, memic_start_addr, addr); + MLX5_SET(dealloc_memic_in, in, memic_size, length); + + err = mlx5_cmd_exec_in(dev, dealloc_memic, in); + if (err) + return; + + spin_lock(&dm->lock); + bitmap_clear(dm->memic_alloc_pages, + start_page_idx, num_pages); + spin_unlock(&dm->lock); +} + +void mlx5_cmd_dealloc_memic_op(struct mlx5_dm *dm, phys_addr_t addr, + u8 operation) +{ + u32 in[MLX5_ST_SZ_DW(modify_memic_in)] = {}; + struct mlx5_core_dev *dev = dm->dev; + + MLX5_SET(modify_memic_in, in, opcode, MLX5_CMD_OP_MODIFY_MEMIC); + MLX5_SET(modify_memic_in, in, op_mod, MLX5_MODIFY_MEMIC_OP_MOD_DEALLOC); + MLX5_SET(modify_memic_in, in, memic_operation_type, operation); + MLX5_SET64(modify_memic_in, in, memic_start_addr, addr - dev->bar_addr); + + mlx5_cmd_exec_in(dev, modify_memic, in); +} + +static int mlx5_cmd_alloc_memic_op(struct mlx5_dm *dm, phys_addr_t addr, + u8 operation, phys_addr_t *op_addr) +{ + u32 out[MLX5_ST_SZ_DW(modify_memic_out)] = {}; + u32 in[MLX5_ST_SZ_DW(modify_memic_in)] = {}; + struct mlx5_core_dev *dev = dm->dev; + int err; + + MLX5_SET(modify_memic_in, in, opcode, MLX5_CMD_OP_MODIFY_MEMIC); + MLX5_SET(modify_memic_in, in, op_mod, MLX5_MODIFY_MEMIC_OP_MOD_ALLOC); + MLX5_SET(modify_memic_in, in, memic_operation_type, operation); + MLX5_SET64(modify_memic_in, in, memic_start_addr, addr - dev->bar_addr); + + err = mlx5_cmd_exec_inout(dev, modify_memic, in, out); + if (err) + return err; + + *op_addr = dev->bar_addr + + MLX5_GET64(modify_memic_out, out, memic_operation_addr); + return 0; +} + +static int add_dm_mmap_entry(struct ib_ucontext *context, + struct mlx5_user_mmap_entry *mentry, u8 mmap_flag, + size_t size, u64 address) +{ + mentry->mmap_flag = mmap_flag; + mentry->address = address; + + return rdma_user_mmap_entry_insert_range( + context, &mentry->rdma_entry, size, + MLX5_IB_MMAP_DEVICE_MEM << 16, + (MLX5_IB_MMAP_DEVICE_MEM << 16) + (1UL << 16) - 1); +} + +static void mlx5_ib_dm_memic_free(struct kref *kref) +{ + struct mlx5_ib_dm_memic *dm = + container_of(kref, struct mlx5_ib_dm_memic, ref); + struct mlx5_ib_dev *dev = to_mdev(dm->base.ibdm.device); + + mlx5_cmd_dealloc_memic(&dev->dm, dm->base.dev_addr, dm->base.size); + kfree(dm); +} + +static int copy_op_to_user(struct mlx5_ib_dm_op_entry *op_entry, + struct uverbs_attr_bundle *attrs) +{ + u64 start_offset; + u16 page_idx; + int err; + + page_idx = op_entry->mentry.rdma_entry.start_pgoff & 0xFFFF; + start_offset = op_entry->op_addr & ~PAGE_MASK; + err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_PAGE_INDEX, + &page_idx, sizeof(page_idx)); + if (err) + return err; + + return uverbs_copy_to(attrs, + MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_START_OFFSET, + &start_offset, sizeof(start_offset)); +} + +static int map_existing_op(struct mlx5_ib_dm_memic *dm, u8 op, + struct uverbs_attr_bundle *attrs) +{ + struct mlx5_ib_dm_op_entry *op_entry; + + op_entry = xa_load(&dm->ops, op); + if (!op_entry) + return -ENOENT; + + return copy_op_to_user(op_entry, attrs); +} + +static int UVERBS_HANDLER(MLX5_IB_METHOD_DM_MAP_OP_ADDR)( + struct uverbs_attr_bundle *attrs) +{ + struct ib_uobject *uobj = uverbs_attr_get_uobject( + attrs, MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_HANDLE); + struct mlx5_ib_dev *dev = to_mdev(uobj->context->device); + struct ib_dm *ibdm = uobj->object; + struct mlx5_ib_dm_memic *dm = to_memic(ibdm); + struct mlx5_ib_dm_op_entry *op_entry; + int err; + u8 op; + + err = uverbs_copy_from(&op, attrs, MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_OP); + if (err) + return err; + + if (!(MLX5_CAP_DEV_MEM(dev->mdev, memic_operations) & BIT(op))) + return -EOPNOTSUPP; + + mutex_lock(&dm->ops_xa_lock); + err = map_existing_op(dm, op, attrs); + if (!err || err != -ENOENT) + goto err_unlock; + + op_entry = kzalloc(sizeof(*op_entry), GFP_KERNEL); + if (!op_entry) + goto err_unlock; + + err = mlx5_cmd_alloc_memic_op(&dev->dm, dm->base.dev_addr, op, + &op_entry->op_addr); + if (err) { + kfree(op_entry); + goto err_unlock; + } + op_entry->op = op; + op_entry->dm = dm; + + err = add_dm_mmap_entry(uobj->context, &op_entry->mentry, + MLX5_IB_MMAP_TYPE_MEMIC_OP, dm->base.size, + op_entry->op_addr & PAGE_MASK); + if (err) { + mlx5_cmd_dealloc_memic_op(&dev->dm, dm->base.dev_addr, op); + kfree(op_entry); + goto err_unlock; + } + /* From this point, entry will be freed by mmap_free */ + kref_get(&dm->ref); + + err = copy_op_to_user(op_entry, attrs); + if (err) + goto err_remove; + + err = xa_insert(&dm->ops, op, op_entry, GFP_KERNEL); + if (err) + goto err_remove; + mutex_unlock(&dm->ops_xa_lock); + + return 0; + +err_remove: + rdma_user_mmap_entry_remove(&op_entry->mentry.rdma_entry); +err_unlock: + mutex_unlock(&dm->ops_xa_lock); + + return err; +} + +static struct ib_dm *handle_alloc_dm_memic(struct ib_ucontext *ctx, + struct ib_dm_alloc_attr *attr, + struct uverbs_attr_bundle *attrs) +{ + struct mlx5_dm *dm_db = &to_mdev(ctx->device)->dm; + struct mlx5_ib_dm_memic *dm; + u64 start_offset; + u16 page_idx; + int err; + u64 address; + + if (!MLX5_CAP_DEV_MEM(dm_db->dev, memic)) + return ERR_PTR(-EOPNOTSUPP); + + dm = kzalloc(sizeof(*dm), GFP_KERNEL); + if (!dm) + return ERR_PTR(-ENOMEM); + + dm->base.type = MLX5_IB_UAPI_DM_TYPE_MEMIC; + dm->base.size = roundup(attr->length, MLX5_MEMIC_BASE_SIZE); + dm->base.ibdm.device = ctx->device; + + kref_init(&dm->ref); + xa_init(&dm->ops); + mutex_init(&dm->ops_xa_lock); + dm->req_length = attr->length; + + err = mlx5_cmd_alloc_memic(dm_db, &dm->base.dev_addr, + dm->base.size, attr->alignment); + if (err) { + kfree(dm); + return ERR_PTR(err); + } + + address = dm->base.dev_addr & PAGE_MASK; + err = add_dm_mmap_entry(ctx, &dm->mentry, MLX5_IB_MMAP_TYPE_MEMIC, + dm->base.size, address); + if (err) { + mlx5_cmd_dealloc_memic(dm_db, dm->base.dev_addr, dm->base.size); + kfree(dm); + return ERR_PTR(err); + } + + page_idx = dm->mentry.rdma_entry.start_pgoff & 0xFFFF; + err = uverbs_copy_to(attrs, MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX, + &page_idx, sizeof(page_idx)); + if (err) + goto err_copy; + + start_offset = dm->base.dev_addr & ~PAGE_MASK; + err = uverbs_copy_to(attrs, + MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET, + &start_offset, sizeof(start_offset)); + if (err) + goto err_copy; + + return &dm->base.ibdm; + +err_copy: + rdma_user_mmap_entry_remove(&dm->mentry.rdma_entry); + return ERR_PTR(err); +} + +static enum mlx5_sw_icm_type get_icm_type(int uapi_type) +{ + return uapi_type == MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM ? + MLX5_SW_ICM_TYPE_STEERING : + MLX5_SW_ICM_TYPE_HEADER_MODIFY; +} + +static struct ib_dm *handle_alloc_dm_sw_icm(struct ib_ucontext *ctx, + struct ib_dm_alloc_attr *attr, + struct uverbs_attr_bundle *attrs, + int type) +{ + struct mlx5_core_dev *dev = to_mdev(ctx->device)->mdev; + enum mlx5_sw_icm_type icm_type = get_icm_type(type); + struct mlx5_ib_dm_icm *dm; + u64 act_size; + int err; + + dm = kzalloc(sizeof(*dm), GFP_KERNEL); + if (!dm) + return ERR_PTR(-ENOMEM); + + dm->base.type = type; + dm->base.ibdm.device = ctx->device; + + if (!capable(CAP_SYS_RAWIO) || !capable(CAP_NET_RAW)) { + err = -EPERM; + goto free; + } + + if (!(MLX5_CAP_FLOWTABLE_NIC_RX(dev, sw_owner) || + MLX5_CAP_FLOWTABLE_NIC_TX(dev, sw_owner) || + MLX5_CAP_FLOWTABLE_NIC_RX(dev, sw_owner_v2) || + MLX5_CAP_FLOWTABLE_NIC_TX(dev, sw_owner_v2))) { + err = -EOPNOTSUPP; + goto free; + } + + /* Allocation size must a multiple of the basic block size + * and a power of 2. + */ + act_size = round_up(attr->length, MLX5_SW_ICM_BLOCK_SIZE(dev)); + act_size = roundup_pow_of_two(act_size); + + dm->base.size = act_size; + err = mlx5_dm_sw_icm_alloc(dev, icm_type, act_size, attr->alignment, + to_mucontext(ctx)->devx_uid, + &dm->base.dev_addr, &dm->obj_id); + if (err) + goto free; + + err = uverbs_copy_to(attrs, MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET, + &dm->base.dev_addr, sizeof(dm->base.dev_addr)); + if (err) { + mlx5_dm_sw_icm_dealloc(dev, icm_type, dm->base.size, + to_mucontext(ctx)->devx_uid, + dm->base.dev_addr, dm->obj_id); + goto free; + } + return &dm->base.ibdm; +free: + kfree(dm); + return ERR_PTR(err); +} + +struct ib_dm *mlx5_ib_alloc_dm(struct ib_device *ibdev, + struct ib_ucontext *context, + struct ib_dm_alloc_attr *attr, + struct uverbs_attr_bundle *attrs) +{ + enum mlx5_ib_uapi_dm_type type; + int err; + + err = uverbs_get_const_default(&type, attrs, + MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE, + MLX5_IB_UAPI_DM_TYPE_MEMIC); + if (err) + return ERR_PTR(err); + + mlx5_ib_dbg(to_mdev(ibdev), "alloc_dm req: dm_type=%d user_length=0x%llx log_alignment=%d\n", + type, attr->length, attr->alignment); + + switch (type) { + case MLX5_IB_UAPI_DM_TYPE_MEMIC: + return handle_alloc_dm_memic(context, attr, attrs); + case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM: + return handle_alloc_dm_sw_icm(context, attr, attrs, type); + case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM: + return handle_alloc_dm_sw_icm(context, attr, attrs, type); + default: + return ERR_PTR(-EOPNOTSUPP); + } +} + +static void dm_memic_remove_ops(struct mlx5_ib_dm_memic *dm) +{ + struct mlx5_ib_dm_op_entry *entry; + unsigned long idx; + + mutex_lock(&dm->ops_xa_lock); + xa_for_each(&dm->ops, idx, entry) { + xa_erase(&dm->ops, idx); + rdma_user_mmap_entry_remove(&entry->mentry.rdma_entry); + } + mutex_unlock(&dm->ops_xa_lock); +} + +static void mlx5_dm_memic_dealloc(struct mlx5_ib_dm_memic *dm) +{ + dm_memic_remove_ops(dm); + rdma_user_mmap_entry_remove(&dm->mentry.rdma_entry); +} + +static int mlx5_dm_icm_dealloc(struct mlx5_ib_ucontext *ctx, + struct mlx5_ib_dm_icm *dm) +{ + enum mlx5_sw_icm_type type = get_icm_type(dm->base.type); + struct mlx5_core_dev *dev = to_mdev(dm->base.ibdm.device)->mdev; + int err; + + err = mlx5_dm_sw_icm_dealloc(dev, type, dm->base.size, ctx->devx_uid, + dm->base.dev_addr, dm->obj_id); + if (!err) + kfree(dm); + return 0; +} + +static int mlx5_ib_dealloc_dm(struct ib_dm *ibdm, + struct uverbs_attr_bundle *attrs) +{ + struct mlx5_ib_ucontext *ctx = rdma_udata_to_drv_context( + &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext); + struct mlx5_ib_dm *dm = to_mdm(ibdm); + + switch (dm->type) { + case MLX5_IB_UAPI_DM_TYPE_MEMIC: + mlx5_dm_memic_dealloc(to_memic(ibdm)); + return 0; + case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM: + case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM: + return mlx5_dm_icm_dealloc(ctx, to_icm(ibdm)); + default: + return -EOPNOTSUPP; + } +} + +static int UVERBS_HANDLER(MLX5_IB_METHOD_DM_QUERY)( + struct uverbs_attr_bundle *attrs) +{ + struct ib_dm *ibdm = + uverbs_attr_get_obj(attrs, MLX5_IB_ATTR_QUERY_DM_REQ_HANDLE); + struct mlx5_ib_dm *dm = to_mdm(ibdm); + struct mlx5_ib_dm_memic *memic; + u64 start_offset; + u16 page_idx; + int err; + + if (dm->type != MLX5_IB_UAPI_DM_TYPE_MEMIC) + return -EOPNOTSUPP; + + memic = to_memic(ibdm); + page_idx = memic->mentry.rdma_entry.start_pgoff & 0xFFFF; + err = uverbs_copy_to(attrs, MLX5_IB_ATTR_QUERY_DM_RESP_PAGE_INDEX, + &page_idx, sizeof(page_idx)); + if (err) + return err; + + start_offset = memic->base.dev_addr & ~PAGE_MASK; + err = uverbs_copy_to(attrs, MLX5_IB_ATTR_QUERY_DM_RESP_START_OFFSET, + &start_offset, sizeof(start_offset)); + if (err) + return err; + + return uverbs_copy_to(attrs, MLX5_IB_ATTR_QUERY_DM_RESP_LENGTH, + &memic->req_length, + sizeof(memic->req_length)); +} + +void mlx5_ib_dm_mmap_free(struct mlx5_ib_dev *dev, + struct mlx5_user_mmap_entry *mentry) +{ + struct mlx5_ib_dm_op_entry *op_entry; + struct mlx5_ib_dm_memic *mdm; + + switch (mentry->mmap_flag) { + case MLX5_IB_MMAP_TYPE_MEMIC: + mdm = container_of(mentry, struct mlx5_ib_dm_memic, mentry); + kref_put(&mdm->ref, mlx5_ib_dm_memic_free); + break; + case MLX5_IB_MMAP_TYPE_MEMIC_OP: + op_entry = container_of(mentry, struct mlx5_ib_dm_op_entry, + mentry); + mdm = op_entry->dm; + mlx5_cmd_dealloc_memic_op(&dev->dm, mdm->base.dev_addr, + op_entry->op); + kfree(op_entry); + kref_put(&mdm->ref, mlx5_ib_dm_memic_free); + break; + default: + WARN_ON(true); + } +} + +DECLARE_UVERBS_NAMED_METHOD( + MLX5_IB_METHOD_DM_QUERY, + UVERBS_ATTR_IDR(MLX5_IB_ATTR_QUERY_DM_REQ_HANDLE, UVERBS_OBJECT_DM, + UVERBS_ACCESS_READ, UA_MANDATORY), + UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_QUERY_DM_RESP_START_OFFSET, + UVERBS_ATTR_TYPE(u64), UA_MANDATORY), + UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_QUERY_DM_RESP_PAGE_INDEX, + UVERBS_ATTR_TYPE(u16), UA_MANDATORY), + UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_QUERY_DM_RESP_LENGTH, + UVERBS_ATTR_TYPE(u64), UA_MANDATORY)); + +ADD_UVERBS_ATTRIBUTES_SIMPLE( + mlx5_ib_dm, UVERBS_OBJECT_DM, UVERBS_METHOD_DM_ALLOC, + UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET, + UVERBS_ATTR_TYPE(u64), UA_MANDATORY), + UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX, + UVERBS_ATTR_TYPE(u16), UA_OPTIONAL), + UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE, + enum mlx5_ib_uapi_dm_type, UA_OPTIONAL)); + +DECLARE_UVERBS_NAMED_METHOD( + MLX5_IB_METHOD_DM_MAP_OP_ADDR, + UVERBS_ATTR_IDR(MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_HANDLE, + UVERBS_OBJECT_DM, + UVERBS_ACCESS_READ, + UA_MANDATORY), + UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_OP, + UVERBS_ATTR_TYPE(u8), + UA_MANDATORY), + UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_START_OFFSET, + UVERBS_ATTR_TYPE(u64), + UA_MANDATORY), + UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_PAGE_INDEX, + UVERBS_ATTR_TYPE(u16), + UA_OPTIONAL)); + +DECLARE_UVERBS_GLOBAL_METHODS(UVERBS_OBJECT_DM, + &UVERBS_METHOD(MLX5_IB_METHOD_DM_MAP_OP_ADDR), + &UVERBS_METHOD(MLX5_IB_METHOD_DM_QUERY)); + +const struct uapi_definition mlx5_ib_dm_defs[] = { + UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DM, &mlx5_ib_dm), + UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_DM), + {}, +}; + +const struct ib_device_ops mlx5_ib_dev_dm_ops = { + .alloc_dm = mlx5_ib_alloc_dm, + .dealloc_dm = mlx5_ib_dealloc_dm, + .reg_dm_mr = mlx5_ib_reg_dm_mr, +}; diff --git a/drivers/infiniband/hw/mlx5/dm.h b/drivers/infiniband/hw/mlx5/dm.h new file mode 100644 index 000000000000..9674a80d8d70 --- /dev/null +++ b/drivers/infiniband/hw/mlx5/dm.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* + * Copyright (c) 2021, Mellanox Technologies inc. All rights reserved. + */ + +#ifndef _MLX5_IB_DM_H +#define _MLX5_IB_DM_H + +#include "mlx5_ib.h" + +extern const struct ib_device_ops mlx5_ib_dev_dm_ops; +extern const struct uapi_definition mlx5_ib_dm_defs[]; + +struct mlx5_ib_dm { + struct ib_dm ibdm; + u32 type; + phys_addr_t dev_addr; + size_t size; +}; + +struct mlx5_ib_dm_op_entry { + struct mlx5_user_mmap_entry mentry; + phys_addr_t op_addr; + struct mlx5_ib_dm_memic *dm; + u8 op; +}; + +struct mlx5_ib_dm_memic { + struct mlx5_ib_dm base; + struct mlx5_user_mmap_entry mentry; + struct xarray ops; + struct mutex ops_xa_lock; + struct kref ref; + size_t req_length; +}; + +struct mlx5_ib_dm_icm { + struct mlx5_ib_dm base; + u32 obj_id; +}; + +static inline struct mlx5_ib_dm *to_mdm(struct ib_dm *ibdm) +{ + return container_of(ibdm, struct mlx5_ib_dm, ibdm); +} + +static inline struct mlx5_ib_dm_memic *to_memic(struct ib_dm *ibdm) +{ + return container_of(ibdm, struct mlx5_ib_dm_memic, base.ibdm); +} + +static inline struct mlx5_ib_dm_icm *to_icm(struct ib_dm *ibdm) +{ + return container_of(ibdm, struct mlx5_ib_dm_icm, base.ibdm); +} + +struct ib_dm *mlx5_ib_alloc_dm(struct ib_device *ibdev, + struct ib_ucontext *context, + struct ib_dm_alloc_attr *attr, + struct uverbs_attr_bundle *attrs); +void mlx5_ib_dm_mmap_free(struct mlx5_ib_dev *dev, + struct mlx5_user_mmap_entry *mentry); +void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, + u64 length); +void mlx5_cmd_dealloc_memic_op(struct mlx5_dm *dm, phys_addr_t addr, + u8 operation); + +#endif /* _MLX5_IB_DM_H */ diff --git a/drivers/infiniband/hw/mlx5/fs.c b/drivers/infiniband/hw/mlx5/fs.c index 01370d9a871a..2fc6a60c4e77 100644 --- a/drivers/infiniband/hw/mlx5/fs.c +++ b/drivers/infiniband/hw/mlx5/fs.c @@ -1528,8 +1528,8 @@ static struct mlx5_ib_flow_handler *raw_fs_rule_add( dst_num++; } - handler = _create_raw_flow_rule(dev, ft_prio, dst, fs_matcher, - flow_context, flow_act, + handler = _create_raw_flow_rule(dev, ft_prio, dst_num ? dst : NULL, + fs_matcher, flow_context, flow_act, cmd_in, inlen, dst_num); if (IS_ERR(handler)) { @@ -1885,8 +1885,9 @@ static int get_dests(struct uverbs_attr_bundle *attrs, else *dest_id = mqp->raw_packet_qp.rq.tirn; *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; - } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS || - fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) { + } else if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS || + fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) && + !(*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)) { *dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT; } diff --git a/drivers/infiniband/hw/mlx5/ib_rep.c b/drivers/infiniband/hw/mlx5/ib_rep.c index db5de720bb12..b25e0b33a11a 100644 --- a/drivers/infiniband/hw/mlx5/ib_rep.c +++ b/drivers/infiniband/hw/mlx5/ib_rep.c @@ -29,7 +29,7 @@ mlx5_ib_set_vport_rep(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep) static int mlx5_ib_vport_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep) { - int num_ports = mlx5_eswitch_get_total_vports(dev); + u32 num_ports = mlx5_eswitch_get_total_vports(dev); const struct mlx5_ib_profile *profile; struct mlx5_ib_dev *ibdev; int vport_index; @@ -110,7 +110,7 @@ struct net_device *mlx5_ib_get_rep_netdev(struct mlx5_eswitch *esw, struct mlx5_flow_handle *create_flow_rule_vport_sq(struct mlx5_ib_dev *dev, struct mlx5_ib_sq *sq, - u16 port) + u32 port) { struct mlx5_eswitch *esw = dev->mdev->priv.eswitch; struct mlx5_eswitch_rep *rep; diff --git a/drivers/infiniband/hw/mlx5/ib_rep.h b/drivers/infiniband/hw/mlx5/ib_rep.h index ce1dcb105dbd..9c55e5c528b4 100644 --- a/drivers/infiniband/hw/mlx5/ib_rep.h +++ b/drivers/infiniband/hw/mlx5/ib_rep.h @@ -16,7 +16,7 @@ int mlx5r_rep_init(void); void mlx5r_rep_cleanup(void); struct mlx5_flow_handle *create_flow_rule_vport_sq(struct mlx5_ib_dev *dev, struct mlx5_ib_sq *sq, - u16 port); + u32 port); struct net_device *mlx5_ib_get_rep_netdev(struct mlx5_eswitch *esw, u16 vport_num); #else /* CONFIG_MLX5_ESWITCH */ @@ -25,7 +25,7 @@ static inline void mlx5r_rep_cleanup(void) {} static inline struct mlx5_flow_handle *create_flow_rule_vport_sq(struct mlx5_ib_dev *dev, struct mlx5_ib_sq *sq, - u16 port) + u32 port) { return NULL; } diff --git a/drivers/infiniband/hw/mlx5/ib_virt.c b/drivers/infiniband/hw/mlx5/ib_virt.c index 46b2d370fb3f..f2f62875d072 100644 --- a/drivers/infiniband/hw/mlx5/ib_virt.c +++ b/drivers/infiniband/hw/mlx5/ib_virt.c @@ -48,7 +48,7 @@ static inline u32 mlx_to_net_policy(enum port_state_policy mlx_policy) } } -int mlx5_ib_get_vf_config(struct ib_device *device, int vf, u8 port, +int mlx5_ib_get_vf_config(struct ib_device *device, int vf, u32 port, struct ifla_vf_info *info) { struct mlx5_ib_dev *dev = to_mdev(device); @@ -91,7 +91,7 @@ static inline enum port_state_policy net_to_mlx_policy(int policy) } int mlx5_ib_set_vf_link_state(struct ib_device *device, int vf, - u8 port, int state) + u32 port, int state) { struct mlx5_ib_dev *dev = to_mdev(device); struct mlx5_core_dev *mdev = dev->mdev; @@ -119,7 +119,7 @@ out: } int mlx5_ib_get_vf_stats(struct ib_device *device, int vf, - u8 port, struct ifla_vf_stats *stats) + u32 port, struct ifla_vf_stats *stats) { int out_sz = MLX5_ST_SZ_BYTES(query_vport_counter_out); struct mlx5_core_dev *mdev; @@ -149,7 +149,8 @@ ex: return err; } -static int set_vf_node_guid(struct ib_device *device, int vf, u8 port, u64 guid) +static int set_vf_node_guid(struct ib_device *device, int vf, u32 port, + u64 guid) { struct mlx5_ib_dev *dev = to_mdev(device); struct mlx5_core_dev *mdev = dev->mdev; @@ -172,7 +173,8 @@ static int set_vf_node_guid(struct ib_device *device, int vf, u8 port, u64 guid) return err; } -static int set_vf_port_guid(struct ib_device *device, int vf, u8 port, u64 guid) +static int set_vf_port_guid(struct ib_device *device, int vf, u32 port, + u64 guid) { struct mlx5_ib_dev *dev = to_mdev(device); struct mlx5_core_dev *mdev = dev->mdev; @@ -195,7 +197,7 @@ static int set_vf_port_guid(struct ib_device *device, int vf, u8 port, u64 guid) return err; } -int mlx5_ib_set_vf_guid(struct ib_device *device, int vf, u8 port, +int mlx5_ib_set_vf_guid(struct ib_device *device, int vf, u32 port, u64 guid, int type) { if (type == IFLA_VF_IB_NODE_GUID) @@ -206,7 +208,7 @@ int mlx5_ib_set_vf_guid(struct ib_device *device, int vf, u8 port, return -EINVAL; } -int mlx5_ib_get_vf_guid(struct ib_device *device, int vf, u8 port, +int mlx5_ib_get_vf_guid(struct ib_device *device, int vf, u32 port, struct ifla_vf_guid *node_guid, struct ifla_vf_guid *port_guid) { diff --git a/drivers/infiniband/hw/mlx5/mad.c b/drivers/infiniband/hw/mlx5/mad.c index 652c6ccf1881..ec242a5a17a3 100644 --- a/drivers/infiniband/hw/mlx5/mad.c +++ b/drivers/infiniband/hw/mlx5/mad.c @@ -42,7 +42,7 @@ enum { MLX5_IB_VENDOR_CLASS2 = 0xa }; -static bool can_do_mad_ifc(struct mlx5_ib_dev *dev, u8 port_num, +static bool can_do_mad_ifc(struct mlx5_ib_dev *dev, u32 port_num, struct ib_mad *in_mad) { if (in_mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_SUBN_LID_ROUTED && @@ -52,7 +52,7 @@ static bool can_do_mad_ifc(struct mlx5_ib_dev *dev, u8 port_num, } static int mlx5_MAD_IFC(struct mlx5_ib_dev *dev, int ignore_mkey, - int ignore_bkey, u8 port, const struct ib_wc *in_wc, + int ignore_bkey, u32 port, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const void *in_mad, void *response_mad) { @@ -147,12 +147,12 @@ static void pma_cnt_assign(struct ib_pma_portcounters *pma_cnt, vl_15_dropped); } -static int process_pma_cmd(struct mlx5_ib_dev *dev, u8 port_num, +static int process_pma_cmd(struct mlx5_ib_dev *dev, u32 port_num, const struct ib_mad *in_mad, struct ib_mad *out_mad) { struct mlx5_core_dev *mdev; bool native_port = true; - u8 mdev_port_num; + u32 mdev_port_num; void *out_cnt; int err; @@ -216,7 +216,7 @@ done: return err; } -int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, +int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const struct ib_mad *in, struct ib_mad *out, size_t *out_mad_size, u16 *out_mad_pkey_index) @@ -444,7 +444,7 @@ out: return err; } -int mlx5_query_mad_ifc_pkey(struct ib_device *ibdev, u8 port, u16 index, +int mlx5_query_mad_ifc_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) { struct ib_smp *in_mad = NULL; @@ -473,7 +473,7 @@ out: return err; } -int mlx5_query_mad_ifc_gids(struct ib_device *ibdev, u8 port, int index, +int mlx5_query_mad_ifc_gids(struct ib_device *ibdev, u32 port, int index, union ib_gid *gid) { struct ib_smp *in_mad = NULL; @@ -513,7 +513,7 @@ out: return err; } -int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u8 port, +int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props) { struct mlx5_ib_dev *dev = to_mdev(ibdev); diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 7a7f6ccd02a5..6d1dd09a4388 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -34,6 +34,7 @@ #include "ib_rep.h" #include "cmd.h" #include "devx.h" +#include "dm.h" #include "fs.h" #include "srq.h" #include "qp.h" @@ -42,6 +43,7 @@ #include "counters.h" #include <linux/mlx5/accel.h> #include <rdma/uverbs_std_types.h> +#include <rdma/uverbs_ioctl.h> #include <rdma/mlx5_user_ioctl_verbs.h> #include <rdma/mlx5_user_ioctl_cmds.h> #include <rdma/ib_umem_odp.h> @@ -100,7 +102,7 @@ mlx5_port_type_cap_to_rdma_ll(int port_type_cap) } static enum rdma_link_layer -mlx5_ib_port_link_layer(struct ib_device *device, u8 port_num) +mlx5_ib_port_link_layer(struct ib_device *device, u32 port_num) { struct mlx5_ib_dev *dev = to_mdev(device); int port_type_cap = MLX5_CAP_GEN(dev->mdev, port_type); @@ -109,7 +111,7 @@ mlx5_ib_port_link_layer(struct ib_device *device, u8 port_num) } static int get_port_state(struct ib_device *ibdev, - u8 port_num, + u32 port_num, enum ib_port_state *state) { struct ib_port_attr attr; @@ -124,7 +126,7 @@ static int get_port_state(struct ib_device *ibdev, static struct mlx5_roce *mlx5_get_rep_roce(struct mlx5_ib_dev *dev, struct net_device *ndev, - u8 *port_num) + u32 *port_num) { struct net_device *rep_ndev; struct mlx5_ib_port *port; @@ -154,7 +156,7 @@ static int mlx5_netdev_event(struct notifier_block *this, { struct mlx5_roce *roce = container_of(this, struct mlx5_roce, nb); struct net_device *ndev = netdev_notifier_info_to_dev(ptr); - u8 port_num = roce->native_port_num; + u32 port_num = roce->native_port_num; struct mlx5_core_dev *mdev; struct mlx5_ib_dev *ibdev; @@ -233,7 +235,7 @@ done: } static struct net_device *mlx5_ib_get_netdev(struct ib_device *device, - u8 port_num) + u32 port_num) { struct mlx5_ib_dev *ibdev = to_mdev(device); struct net_device *ndev; @@ -261,8 +263,8 @@ out: } struct mlx5_core_dev *mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev *ibdev, - u8 ib_port_num, - u8 *native_port_num) + u32 ib_port_num, + u32 *native_port_num) { enum rdma_link_layer ll = mlx5_ib_port_link_layer(&ibdev->ib_dev, ib_port_num); @@ -296,7 +298,7 @@ struct mlx5_core_dev *mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev *ibdev, return mdev; } -void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *ibdev, u8 port_num) +void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *ibdev, u32 port_num) { enum rdma_link_layer ll = mlx5_ib_port_link_layer(&ibdev->ib_dev, port_num); @@ -452,7 +454,7 @@ static int translate_eth_proto_oper(u32 eth_proto_oper, u16 *active_speed, active_width); } -static int mlx5_query_port_roce(struct ib_device *device, u8 port_num, +static int mlx5_query_port_roce(struct ib_device *device, u32 port_num, struct ib_port_attr *props) { struct mlx5_ib_dev *dev = to_mdev(device); @@ -462,7 +464,7 @@ static int mlx5_query_port_roce(struct ib_device *device, u8 port_num, enum ib_mtu ndev_ib_mtu; bool put_mdev = true; u32 eth_prot_oper; - u8 mdev_port_num; + u32 mdev_port_num; bool ext; int err; @@ -498,7 +500,7 @@ static int mlx5_query_port_roce(struct ib_device *device, u8 port_num, translate_eth_proto_oper(eth_prot_oper, &props->active_speed, &props->active_width, ext); - if (!dev->is_rep && mlx5_is_roce_enabled(mdev)) { + if (!dev->is_rep && dev->mdev->roce.roce_en) { u16 qkey_viol_cntr; props->port_cap_flags |= IB_PORT_CM_SUP; @@ -549,19 +551,19 @@ out: return err; } -static int set_roce_addr(struct mlx5_ib_dev *dev, u8 port_num, +static int set_roce_addr(struct mlx5_ib_dev *dev, u32 port_num, unsigned int index, const union ib_gid *gid, const struct ib_gid_attr *attr) { - enum ib_gid_type gid_type = IB_GID_TYPE_ROCE; + enum ib_gid_type gid_type; u16 vlan_id = 0xffff; u8 roce_version = 0; u8 roce_l3_type = 0; u8 mac[ETH_ALEN]; int ret; + gid_type = attr->gid_type; if (gid) { - gid_type = attr->gid_type; ret = rdma_read_gid_l2_fields(attr, &vlan_id, &mac[0]); if (ret) return ret; @@ -573,7 +575,7 @@ static int set_roce_addr(struct mlx5_ib_dev *dev, u8 port_num, break; case IB_GID_TYPE_ROCE_UDP_ENCAP: roce_version = MLX5_ROCE_VERSION_2; - if (ipv6_addr_v4mapped((void *)gid)) + if (gid && ipv6_addr_v4mapped((void *)gid)) roce_l3_type = MLX5_ROCE_L3_TYPE_IPV4; else roce_l3_type = MLX5_ROCE_L3_TYPE_IPV6; @@ -600,7 +602,7 @@ static int mlx5_ib_del_gid(const struct ib_gid_attr *attr, __always_unused void **context) { return set_roce_addr(to_mdev(attr->device), attr->port_num, - attr->index, NULL, NULL); + attr->index, NULL, attr); } __be16 mlx5_get_roce_udp_sport_min(const struct mlx5_ib_dev *dev, @@ -1267,7 +1269,7 @@ static int translate_max_vl_num(struct ib_device *ibdev, u8 vl_hw_cap, return 0; } -static int mlx5_query_hca_port(struct ib_device *ibdev, u8 port, +static int mlx5_query_hca_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props) { struct mlx5_ib_dev *dev = to_mdev(ibdev); @@ -1335,7 +1337,7 @@ out: return err; } -int mlx5_ib_query_port(struct ib_device *ibdev, u8 port, +int mlx5_ib_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props) { unsigned int count; @@ -1380,13 +1382,13 @@ int mlx5_ib_query_port(struct ib_device *ibdev, u8 port, return ret; } -static int mlx5_ib_rep_query_port(struct ib_device *ibdev, u8 port, +static int mlx5_ib_rep_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props) { return mlx5_query_port_roce(ibdev, port, props); } -static int mlx5_ib_rep_query_pkey(struct ib_device *ibdev, u8 port, u16 index, +static int mlx5_ib_rep_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) { /* Default special Pkey for representor device port as per the @@ -1396,7 +1398,7 @@ static int mlx5_ib_rep_query_pkey(struct ib_device *ibdev, u8 port, u16 index, return 0; } -static int mlx5_ib_query_gid(struct ib_device *ibdev, u8 port, int index, +static int mlx5_ib_query_gid(struct ib_device *ibdev, u32 port, int index, union ib_gid *gid) { struct mlx5_ib_dev *dev = to_mdev(ibdev); @@ -1415,13 +1417,13 @@ static int mlx5_ib_query_gid(struct ib_device *ibdev, u8 port, int index, } -static int mlx5_query_hca_nic_pkey(struct ib_device *ibdev, u8 port, +static int mlx5_query_hca_nic_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) { struct mlx5_ib_dev *dev = to_mdev(ibdev); struct mlx5_core_dev *mdev; bool put_mdev = true; - u8 mdev_port_num; + u32 mdev_port_num; int err; mdev = mlx5_ib_get_native_port_mdev(dev, port, &mdev_port_num); @@ -1442,7 +1444,7 @@ static int mlx5_query_hca_nic_pkey(struct ib_device *ibdev, u8 port, return err; } -static int mlx5_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index, +static int mlx5_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) { switch (mlx5_get_vport_access_method(ibdev)) { @@ -1486,12 +1488,12 @@ static int mlx5_ib_modify_device(struct ib_device *ibdev, int mask, return err; } -static int set_port_caps_atomic(struct mlx5_ib_dev *dev, u8 port_num, u32 mask, +static int set_port_caps_atomic(struct mlx5_ib_dev *dev, u32 port_num, u32 mask, u32 value) { struct mlx5_hca_vport_context ctx = {}; struct mlx5_core_dev *mdev; - u8 mdev_port_num; + u32 mdev_port_num; int err; mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &mdev_port_num); @@ -1520,7 +1522,7 @@ out: return err; } -static int mlx5_ib_modify_port(struct ib_device *ibdev, u8 port, int mask, +static int mlx5_ib_modify_port(struct ib_device *ibdev, u32 port, int mask, struct ib_port_modify *props) { struct mlx5_ib_dev *dev = to_mdev(ibdev); @@ -1929,7 +1931,7 @@ uar_done: print_lib_caps(dev, context->lib_caps); if (mlx5_ib_lag_should_assign_affinity(dev)) { - u8 port = mlx5_core_native_port_num(dev->mdev) - 1; + u32 port = mlx5_core_native_port_num(dev->mdev) - 1; atomic_set(&context->tx_port_affinity, atomic_add_return( @@ -2087,14 +2089,11 @@ static void mlx5_ib_mmap_free(struct rdma_user_mmap_entry *entry) struct mlx5_user_mmap_entry *mentry = to_mmmap(entry); struct mlx5_ib_dev *dev = to_mdev(entry->ucontext->device); struct mlx5_var_table *var_table = &dev->var_table; - struct mlx5_ib_dm *mdm; switch (mentry->mmap_flag) { case MLX5_IB_MMAP_TYPE_MEMIC: - mdm = container_of(mentry, struct mlx5_ib_dm, mentry); - mlx5_cmd_dealloc_memic(&dev->dm, mdm->dev_addr, - mdm->size); - kfree(mdm); + case MLX5_IB_MMAP_TYPE_MEMIC_OP: + mlx5_ib_dm_mmap_free(dev, mentry); break; case MLX5_IB_MMAP_TYPE_VAR: mutex_lock(&var_table->bitmap_lock); @@ -2219,19 +2218,6 @@ free_bfreg: return err; } -static int add_dm_mmap_entry(struct ib_ucontext *context, - struct mlx5_ib_dm *mdm, - u64 address) -{ - mdm->mentry.mmap_flag = MLX5_IB_MMAP_TYPE_MEMIC; - mdm->mentry.address = address; - return rdma_user_mmap_entry_insert_range( - context, &mdm->mentry.rdma_entry, - mdm->size, - MLX5_IB_MMAP_DEVICE_MEM << 16, - (MLX5_IB_MMAP_DEVICE_MEM << 16) + (1UL << 16) - 1); -} - static unsigned long mlx5_vma_to_pgoff(struct vm_area_struct *vma) { unsigned long idx; @@ -2333,206 +2319,6 @@ static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vm return 0; } -static inline int check_dm_type_support(struct mlx5_ib_dev *dev, - u32 type) -{ - switch (type) { - case MLX5_IB_UAPI_DM_TYPE_MEMIC: - if (!MLX5_CAP_DEV_MEM(dev->mdev, memic)) - return -EOPNOTSUPP; - break; - case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM: - case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM: - if (!capable(CAP_SYS_RAWIO) || - !capable(CAP_NET_RAW)) - return -EPERM; - - if (!(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner) || - MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, sw_owner) || - MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner_v2) || - MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, sw_owner_v2))) - return -EOPNOTSUPP; - break; - } - - return 0; -} - -static int handle_alloc_dm_memic(struct ib_ucontext *ctx, - struct mlx5_ib_dm *dm, - struct ib_dm_alloc_attr *attr, - struct uverbs_attr_bundle *attrs) -{ - struct mlx5_dm *dm_db = &to_mdev(ctx->device)->dm; - u64 start_offset; - u16 page_idx; - int err; - u64 address; - - dm->size = roundup(attr->length, MLX5_MEMIC_BASE_SIZE); - - err = mlx5_cmd_alloc_memic(dm_db, &dm->dev_addr, - dm->size, attr->alignment); - if (err) - return err; - - address = dm->dev_addr & PAGE_MASK; - err = add_dm_mmap_entry(ctx, dm, address); - if (err) - goto err_dealloc; - - page_idx = dm->mentry.rdma_entry.start_pgoff & 0xFFFF; - err = uverbs_copy_to(attrs, - MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX, - &page_idx, - sizeof(page_idx)); - if (err) - goto err_copy; - - start_offset = dm->dev_addr & ~PAGE_MASK; - err = uverbs_copy_to(attrs, - MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET, - &start_offset, sizeof(start_offset)); - if (err) - goto err_copy; - - return 0; - -err_copy: - rdma_user_mmap_entry_remove(&dm->mentry.rdma_entry); -err_dealloc: - mlx5_cmd_dealloc_memic(dm_db, dm->dev_addr, dm->size); - - return err; -} - -static int handle_alloc_dm_sw_icm(struct ib_ucontext *ctx, - struct mlx5_ib_dm *dm, - struct ib_dm_alloc_attr *attr, - struct uverbs_attr_bundle *attrs, - int type) -{ - struct mlx5_core_dev *dev = to_mdev(ctx->device)->mdev; - u64 act_size; - int err; - - /* Allocation size must a multiple of the basic block size - * and a power of 2. - */ - act_size = round_up(attr->length, MLX5_SW_ICM_BLOCK_SIZE(dev)); - act_size = roundup_pow_of_two(act_size); - - dm->size = act_size; - err = mlx5_dm_sw_icm_alloc(dev, type, act_size, attr->alignment, - to_mucontext(ctx)->devx_uid, &dm->dev_addr, - &dm->icm_dm.obj_id); - if (err) - return err; - - err = uverbs_copy_to(attrs, - MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET, - &dm->dev_addr, sizeof(dm->dev_addr)); - if (err) - mlx5_dm_sw_icm_dealloc(dev, type, dm->size, - to_mucontext(ctx)->devx_uid, dm->dev_addr, - dm->icm_dm.obj_id); - - return err; -} - -struct ib_dm *mlx5_ib_alloc_dm(struct ib_device *ibdev, - struct ib_ucontext *context, - struct ib_dm_alloc_attr *attr, - struct uverbs_attr_bundle *attrs) -{ - struct mlx5_ib_dm *dm; - enum mlx5_ib_uapi_dm_type type; - int err; - - err = uverbs_get_const_default(&type, attrs, - MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE, - MLX5_IB_UAPI_DM_TYPE_MEMIC); - if (err) - return ERR_PTR(err); - - mlx5_ib_dbg(to_mdev(ibdev), "alloc_dm req: dm_type=%d user_length=0x%llx log_alignment=%d\n", - type, attr->length, attr->alignment); - - err = check_dm_type_support(to_mdev(ibdev), type); - if (err) - return ERR_PTR(err); - - dm = kzalloc(sizeof(*dm), GFP_KERNEL); - if (!dm) - return ERR_PTR(-ENOMEM); - - dm->type = type; - - switch (type) { - case MLX5_IB_UAPI_DM_TYPE_MEMIC: - err = handle_alloc_dm_memic(context, dm, - attr, - attrs); - break; - case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM: - err = handle_alloc_dm_sw_icm(context, dm, - attr, attrs, - MLX5_SW_ICM_TYPE_STEERING); - break; - case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM: - err = handle_alloc_dm_sw_icm(context, dm, - attr, attrs, - MLX5_SW_ICM_TYPE_HEADER_MODIFY); - break; - default: - err = -EOPNOTSUPP; - } - - if (err) - goto err_free; - - return &dm->ibdm; - -err_free: - kfree(dm); - return ERR_PTR(err); -} - -int mlx5_ib_dealloc_dm(struct ib_dm *ibdm, struct uverbs_attr_bundle *attrs) -{ - struct mlx5_ib_ucontext *ctx = rdma_udata_to_drv_context( - &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext); - struct mlx5_core_dev *dev = to_mdev(ibdm->device)->mdev; - struct mlx5_ib_dm *dm = to_mdm(ibdm); - int ret; - - switch (dm->type) { - case MLX5_IB_UAPI_DM_TYPE_MEMIC: - rdma_user_mmap_entry_remove(&dm->mentry.rdma_entry); - return 0; - case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM: - ret = mlx5_dm_sw_icm_dealloc(dev, MLX5_SW_ICM_TYPE_STEERING, - dm->size, ctx->devx_uid, dm->dev_addr, - dm->icm_dm.obj_id); - if (ret) - return ret; - break; - case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM: - ret = mlx5_dm_sw_icm_dealloc(dev, MLX5_SW_ICM_TYPE_HEADER_MODIFY, - dm->size, ctx->devx_uid, dm->dev_addr, - dm->icm_dm.obj_id); - if (ret) - return ret; - break; - default: - return -EOPNOTSUPP; - } - - kfree(dm); - - return 0; -} - static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) { struct mlx5_ib_pd *pd = to_mpd(ibpd); @@ -2779,7 +2565,7 @@ static void delay_drop_handler(struct work_struct *work) static void handle_general_event(struct mlx5_ib_dev *ibdev, struct mlx5_eqe *eqe, struct ib_event *ibev) { - u8 port = (eqe->data.port.port >> 4) & 0xf; + u32 port = (eqe->data.port.port >> 4) & 0xf; switch (eqe->sub_type) { case MLX5_GENERAL_SUBTYPE_DELAY_DROP_TIMEOUT: @@ -2795,7 +2581,7 @@ static void handle_general_event(struct mlx5_ib_dev *ibdev, struct mlx5_eqe *eqe static int handle_port_change(struct mlx5_ib_dev *ibdev, struct mlx5_eqe *eqe, struct ib_event *ibev) { - u8 port = (eqe->data.port.port >> 4) & 0xf; + u32 port = (eqe->data.port.port >> 4) & 0xf; ibev->element.port_num = port; @@ -3152,7 +2938,7 @@ static u32 get_core_cap_flags(struct ib_device *ibdev, return ret; } -static int mlx5_port_immutable(struct ib_device *ibdev, u8 port_num, +static int mlx5_port_immutable(struct ib_device *ibdev, u32 port_num, struct ib_port_immutable *immutable) { struct ib_port_attr attr; @@ -3180,7 +2966,7 @@ static int mlx5_port_immutable(struct ib_device *ibdev, u8 port_num, return 0; } -static int mlx5_port_rep_immutable(struct ib_device *ibdev, u8 port_num, +static int mlx5_port_rep_immutable(struct ib_device *ibdev, u32 port_num, struct ib_port_immutable *immutable) { struct ib_port_attr attr; @@ -3252,7 +3038,7 @@ static void mlx5_eth_lag_cleanup(struct mlx5_ib_dev *dev) } } -static int mlx5_add_netdev_notifier(struct mlx5_ib_dev *dev, u8 port_num) +static int mlx5_add_netdev_notifier(struct mlx5_ib_dev *dev, u32 port_num) { int err; @@ -3266,7 +3052,7 @@ static int mlx5_add_netdev_notifier(struct mlx5_ib_dev *dev, u8 port_num) return 0; } -static void mlx5_remove_netdev_notifier(struct mlx5_ib_dev *dev, u8 port_num) +static void mlx5_remove_netdev_notifier(struct mlx5_ib_dev *dev, u32 port_num) { if (dev->port[port_num].roce.nb.notifier_call) { unregister_netdevice_notifier(&dev->port[port_num].roce.nb); @@ -3300,7 +3086,7 @@ static void mlx5_disable_eth(struct mlx5_ib_dev *dev) mlx5_nic_vport_disable_roce(dev->mdev); } -static int mlx5_ib_rn_get_params(struct ib_device *device, u8 port_num, +static int mlx5_ib_rn_get_params(struct ib_device *device, u32 port_num, enum rdma_netdev_t type, struct rdma_netdev_alloc_params *params) { @@ -3352,7 +3138,7 @@ static const struct file_operations fops_delay_drop_timeout = { static void mlx5_ib_unbind_slave_port(struct mlx5_ib_dev *ibdev, struct mlx5_ib_multiport_info *mpi) { - u8 port_num = mlx5_core_native_port_num(mpi->mdev) - 1; + u32 port_num = mlx5_core_native_port_num(mpi->mdev) - 1; struct mlx5_ib_port *port = &ibdev->port[port_num]; int comps; int err; @@ -3398,7 +3184,7 @@ static void mlx5_ib_unbind_slave_port(struct mlx5_ib_dev *ibdev, err = mlx5_nic_vport_unaffiliate_multiport(mpi->mdev); - mlx5_ib_dbg(ibdev, "unaffiliated port %d\n", port_num + 1); + mlx5_ib_dbg(ibdev, "unaffiliated port %u\n", port_num + 1); /* Log an error, still needed to cleanup the pointers and add * it back to the list. */ @@ -3412,14 +3198,14 @@ static void mlx5_ib_unbind_slave_port(struct mlx5_ib_dev *ibdev, static bool mlx5_ib_bind_slave_port(struct mlx5_ib_dev *ibdev, struct mlx5_ib_multiport_info *mpi) { - u8 port_num = mlx5_core_native_port_num(mpi->mdev) - 1; + u32 port_num = mlx5_core_native_port_num(mpi->mdev) - 1; int err; lockdep_assert_held(&mlx5_ib_multiport_mutex); spin_lock(&ibdev->port[port_num].mp.mpi_lock); if (ibdev->port[port_num].mp.mpi) { - mlx5_ib_dbg(ibdev, "port %d already affiliated.\n", + mlx5_ib_dbg(ibdev, "port %u already affiliated.\n", port_num + 1); spin_unlock(&ibdev->port[port_num].mp.mpi_lock); return false; @@ -3455,12 +3241,12 @@ unbind: static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev) { - int port_num = mlx5_core_native_port_num(dev->mdev) - 1; + u32 port_num = mlx5_core_native_port_num(dev->mdev) - 1; enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev, port_num + 1); struct mlx5_ib_multiport_info *mpi; int err; - int i; + u32 i; if (!mlx5_core_is_mp_master(dev->mdev) || ll != IB_LINK_LAYER_ETHERNET) return 0; @@ -3523,10 +3309,10 @@ static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev) static void mlx5_ib_cleanup_multiport_master(struct mlx5_ib_dev *dev) { - int port_num = mlx5_core_native_port_num(dev->mdev) - 1; + u32 port_num = mlx5_core_native_port_num(dev->mdev) - 1; enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev, port_num + 1); - int i; + u32 i; if (!mlx5_core_is_mp_master(dev->mdev) || ll != IB_LINK_LAYER_ETHERNET) return; @@ -3539,7 +3325,8 @@ static void mlx5_ib_cleanup_multiport_master(struct mlx5_ib_dev *dev) kfree(dev->port[i].mp.mpi); dev->port[i].mp.mpi = NULL; } else { - mlx5_ib_dbg(dev, "unbinding port_num: %d\n", i + 1); + mlx5_ib_dbg(dev, "unbinding port_num: %u\n", + i + 1); mlx5_ib_unbind_slave_port(dev, dev->port[i].mp.mpi); } } @@ -3816,20 +3603,6 @@ DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_UAR, &UVERBS_METHOD(MLX5_IB_METHOD_UAR_OBJ_DESTROY)); ADD_UVERBS_ATTRIBUTES_SIMPLE( - mlx5_ib_dm, - UVERBS_OBJECT_DM, - UVERBS_METHOD_DM_ALLOC, - UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET, - UVERBS_ATTR_TYPE(u64), - UA_MANDATORY), - UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX, - UVERBS_ATTR_TYPE(u16), - UA_OPTIONAL), - UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE, - enum mlx5_ib_uapi_dm_type, - UA_OPTIONAL)); - -ADD_UVERBS_ATTRIBUTES_SIMPLE( mlx5_ib_flow_action, UVERBS_OBJECT_FLOW_ACTION, UVERBS_METHOD_FLOW_ACTION_ESP_CREATE, @@ -3851,10 +3624,10 @@ static const struct uapi_definition mlx5_ib_defs[] = { UAPI_DEF_CHAIN(mlx5_ib_flow_defs), UAPI_DEF_CHAIN(mlx5_ib_qos_defs), UAPI_DEF_CHAIN(mlx5_ib_std_types_defs), + UAPI_DEF_CHAIN(mlx5_ib_dm_defs), UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION, &mlx5_ib_flow_action), - UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DM, &mlx5_ib_dm), UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DEVICE, &mlx5_ib_query_context), UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_VAR, UAPI_DEF_IS_OBJ_SUPPORTED(var_is_supported)), @@ -3891,8 +3664,6 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev) dev->port[i].roce.last_port_state = IB_PORT_DOWN; } - mlx5_ib_internal_fill_odp_caps(dev); - err = mlx5_ib_init_multiport_master(dev); if (err) return err; @@ -4032,12 +3803,6 @@ static const struct ib_device_ops mlx5_ib_dev_xrc_ops = { INIT_RDMA_OBJ_SIZE(ib_xrcd, mlx5_ib_xrcd, ibxrcd), }; -static const struct ib_device_ops mlx5_ib_dev_dm_ops = { - .alloc_dm = mlx5_ib_alloc_dm, - .dealloc_dm = mlx5_ib_dealloc_dm, - .reg_dm_mr = mlx5_ib_reg_dm_mr, -}; - static int mlx5_ib_init_var_table(struct mlx5_ib_dev *dev) { struct mlx5_core_dev *mdev = dev->mdev; @@ -4160,7 +3925,7 @@ static int mlx5_ib_roce_init(struct mlx5_ib_dev *dev) struct mlx5_core_dev *mdev = dev->mdev; enum rdma_link_layer ll; int port_type_cap; - u8 port_num = 0; + u32 port_num = 0; int err; port_type_cap = MLX5_CAP_GEN(mdev, port_type); @@ -4173,7 +3938,7 @@ static int mlx5_ib_roce_init(struct mlx5_ib_dev *dev) /* Register only for native ports */ err = mlx5_add_netdev_notifier(dev, port_num); - if (err || dev->is_rep || !mlx5_is_roce_enabled(mdev)) + if (err || dev->is_rep || !mlx5_is_roce_init_enabled(mdev)) /* * We don't enable ETH interface for * 1. IB representors @@ -4197,7 +3962,7 @@ static void mlx5_ib_roce_cleanup(struct mlx5_ib_dev *dev) struct mlx5_core_dev *mdev = dev->mdev; enum rdma_link_layer ll; int port_type_cap; - u8 port_num; + u32 port_num; port_type_cap = MLX5_CAP_GEN(mdev, port_type); ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap); @@ -4710,7 +4475,7 @@ static int mlx5r_probe(struct auxiliary_device *adev, dev->mdev = mdev; dev->num_ports = num_ports; - if (ll == IB_LINK_LAYER_ETHERNET && !mlx5_is_roce_enabled(mdev)) + if (ll == IB_LINK_LAYER_ETHERNET && !mlx5_is_roce_init_enabled(mdev)) profile = &raw_eth_profile; else profile = &pf_profile; diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 88cc26e008fc..e9a3f34a30b8 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -166,6 +166,7 @@ enum mlx5_ib_mmap_type { MLX5_IB_MMAP_TYPE_VAR = 2, MLX5_IB_MMAP_TYPE_UAR_WC = 3, MLX5_IB_MMAP_TYPE_UAR_NC = 4, + MLX5_IB_MMAP_TYPE_MEMIC_OP = 5, }; struct mlx5_bfreg_info { @@ -406,7 +407,7 @@ struct mlx5_ib_qp_base { struct mlx5_ib_qp_trans { struct mlx5_ib_qp_base base; u16 xrcdn; - u8 alt_port; + u32 alt_port; u8 atomic_rd_en; u8 resp_depth; }; @@ -453,7 +454,7 @@ struct mlx5_ib_dct { struct mlx5_ib_gsi_qp { struct ib_qp *rx_qp; - u8 port_num; + u32 port_num; struct ib_qp_cap cap; struct ib_cq *cq; struct mlx5_ib_gsi_wr *outstanding_wrs; @@ -490,7 +491,7 @@ struct mlx5_ib_qp { struct mutex mutex; /* cached variant of create_flags from struct ib_qp_init_attr */ u32 flags; - u8 port; + u32 port; u8 state; int max_inline_data; struct mlx5_bf bf; @@ -547,11 +548,6 @@ static inline const struct mlx5_umr_wr *umr_wr(const struct ib_send_wr *wr) return container_of(wr, struct mlx5_umr_wr, wr); } -struct mlx5_shared_mr_info { - int mr_id; - struct ib_umem *umem; -}; - enum mlx5_ib_cq_pr_flags { MLX5_IB_CQ_PR_FLAGS_CQE_128_PAD = 1 << 0, }; @@ -623,20 +619,6 @@ struct mlx5_user_mmap_entry { u32 page_idx; }; -struct mlx5_ib_dm { - struct ib_dm ibdm; - phys_addr_t dev_addr; - u32 type; - size_t size; - union { - struct { - u32 obj_id; - } icm_dm; - /* other dm types specific params should be added here */ - }; - struct mlx5_user_mmap_entry mentry; -}; - #define MLX5_IB_MTT_PRESENT (MLX5_IB_MTT_READ | MLX5_IB_MTT_WRITE) #define MLX5_IB_DM_MEMIC_ALLOWED_ACCESS (IB_ACCESS_LOCAL_WRITE |\ @@ -654,47 +636,69 @@ struct mlx5_ib_dm { atomic64_add(value, &((mr)->odp_stats.counter_name)) struct mlx5_ib_mr { - struct ib_mr ibmr; - void *descs; - dma_addr_t desc_map; - int ndescs; - int data_length; - int meta_ndescs; - int meta_length; - int max_descs; - int desc_size; - int access_mode; - unsigned int page_shift; - struct mlx5_core_mkey mmkey; - struct ib_umem *umem; - struct mlx5_shared_mr_info *smr_info; - struct list_head list; - struct mlx5_cache_ent *cache_ent; - u32 out[MLX5_ST_SZ_DW(create_mkey_out)]; - struct mlx5_core_sig_ctx *sig; - void *descs_alloc; - int access_flags; /* Needed for rereg MR */ - - struct mlx5_ib_mr *parent; - /* Needed for IB_MR_TYPE_INTEGRITY */ - struct mlx5_ib_mr *pi_mr; - struct mlx5_ib_mr *klm_mr; - struct mlx5_ib_mr *mtt_mr; - u64 data_iova; - u64 pi_iova; - - /* For ODP and implicit */ - struct xarray implicit_children; + struct ib_mr ibmr; + struct mlx5_core_mkey mmkey; + + /* User MR data */ + struct mlx5_cache_ent *cache_ent; + struct ib_umem *umem; + + /* This is zero'd when the MR is allocated */ union { - struct list_head elm; - struct work_struct work; - } odp_destroy; - struct ib_odp_counters odp_stats; - bool is_odp_implicit; + /* Used only while the MR is in the cache */ + struct { + u32 out[MLX5_ST_SZ_DW(create_mkey_out)]; + struct mlx5_async_work cb_work; + /* Cache list element */ + struct list_head list; + }; - struct mlx5_async_work cb_work; + /* Used only by kernel MRs (umem == NULL) */ + struct { + void *descs; + void *descs_alloc; + dma_addr_t desc_map; + int max_descs; + int ndescs; + int desc_size; + int access_mode; + + /* For Kernel IB_MR_TYPE_INTEGRITY */ + struct mlx5_core_sig_ctx *sig; + struct mlx5_ib_mr *pi_mr; + struct mlx5_ib_mr *klm_mr; + struct mlx5_ib_mr *mtt_mr; + u64 data_iova; + u64 pi_iova; + int meta_ndescs; + int meta_length; + int data_length; + }; + + /* Used only by User MRs (umem != NULL) */ + struct { + unsigned int page_shift; + /* Current access_flags */ + int access_flags; + + /* For User ODP */ + struct mlx5_ib_mr *parent; + struct xarray implicit_children; + union { + struct work_struct work; + } odp_destroy; + struct ib_odp_counters odp_stats; + bool is_odp_implicit; + }; + }; }; +/* Zero the fields in the mr that are variant depending on usage */ +static inline void mlx5_clear_mr(struct mlx5_ib_mr *mr) +{ + memset(mr->out, 0, sizeof(*mr) - offsetof(struct mlx5_ib_mr, out)); +} + static inline bool is_odp_mr(struct mlx5_ib_mr *mr) { return IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) && mr->umem && @@ -822,7 +826,7 @@ struct mlx5_roce { atomic_t tx_port_affinity; enum ib_port_state last_port_state; struct mlx5_ib_dev *dev; - u8 native_port_num; + u32 native_port_num; }; struct mlx5_ib_port { @@ -837,7 +841,7 @@ struct mlx5_ib_dbg_param { int offset; struct mlx5_ib_dev *dev; struct dentry *dentry; - u8 port_num; + u32 port_num; }; enum mlx5_ib_dbg_cc_types { @@ -1063,6 +1067,7 @@ struct mlx5_ib_dev { struct mutex slow_path_mutex; struct ib_odp_caps odp_caps; u64 odp_max_size; + struct mutex odp_eq_mutex; struct mlx5_ib_pf_eq odp_pf_eq; struct xarray odp_mkeys; @@ -1170,11 +1175,6 @@ static inline struct mlx5_ib_srq *to_mibsrq(struct mlx5_core_srq *msrq) return container_of(msrq, struct mlx5_ib_srq, msrq); } -static inline struct mlx5_ib_dm *to_mdm(struct ib_dm *ibdm) -{ - return container_of(ibdm, struct mlx5_ib_dm, ibdm); -} - static inline struct mlx5_ib_mr *to_mmr(struct ib_mr *ibmr) { return container_of(ibmr, struct mlx5_ib_mr, ibmr); @@ -1268,8 +1268,7 @@ struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd, struct ib_udata *udata, int access_flags); void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *mr); -void mlx5_ib_fence_odp_mr(struct mlx5_ib_mr *mr); -void mlx5_ib_fence_dmabuf_mr(struct mlx5_ib_mr *mr); +void mlx5_ib_free_odp_mr(struct mlx5_ib_mr *mr); struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, u64 length, u64 virt_addr, int access_flags, struct ib_pd *pd, struct ib_udata *udata); @@ -1285,7 +1284,7 @@ int mlx5_ib_map_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, int data_sg_nents, unsigned int *data_sg_offset, struct scatterlist *meta_sg, int meta_sg_nents, unsigned int *meta_sg_offset); -int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, +int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const struct ib_mad *in, struct ib_mad *out, size_t *out_mad_size, u16 *out_mad_pkey_index); @@ -1300,13 +1299,13 @@ int mlx5_query_mad_ifc_vendor_id(struct ib_device *ibdev, u32 *vendor_id); int mlx5_query_mad_ifc_node_desc(struct mlx5_ib_dev *dev, char *node_desc); int mlx5_query_mad_ifc_node_guid(struct mlx5_ib_dev *dev, __be64 *node_guid); -int mlx5_query_mad_ifc_pkey(struct ib_device *ibdev, u8 port, u16 index, +int mlx5_query_mad_ifc_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey); -int mlx5_query_mad_ifc_gids(struct ib_device *ibdev, u8 port, int index, +int mlx5_query_mad_ifc_gids(struct ib_device *ibdev, u32 port, int index, union ib_gid *gid); -int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u8 port, +int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props); -int mlx5_ib_query_port(struct ib_device *ibdev, u8 port, +int mlx5_ib_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props); void mlx5_ib_populate_pas(struct ib_umem *umem, size_t page_size, __be64 *pas, u64 access_flags); @@ -1317,8 +1316,6 @@ int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev); struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, unsigned int entry, int access_flags); -void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr); -int mlx5_mr_cache_invalidate(struct mlx5_ib_mr *mr); int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask, struct ib_mr_status *mr_status); @@ -1332,18 +1329,13 @@ int mlx5_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table, struct ib_rwq_ind_table_init_attr *init_attr, struct ib_udata *udata); int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table); -struct ib_dm *mlx5_ib_alloc_dm(struct ib_device *ibdev, - struct ib_ucontext *context, - struct ib_dm_alloc_attr *attr, - struct uverbs_attr_bundle *attrs); -int mlx5_ib_dealloc_dm(struct ib_dm *ibdm, struct uverbs_attr_bundle *attrs); struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm, struct ib_dm_mr_attr *attr, struct uverbs_attr_bundle *attrs); #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING -void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev); int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev); +int mlx5r_odp_create_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq); void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *ibdev); int __init mlx5_ib_odp_init(void); void mlx5_ib_odp_cleanup(void); @@ -1357,12 +1349,12 @@ int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr); int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr); #else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ -static inline void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev) +static inline int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev) { return 0; } +static inline int mlx5r_odp_create_eq(struct mlx5_ib_dev *dev, + struct mlx5_ib_pf_eq *eq) { - return; + return 0; } - -static inline int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev) { return 0; } static inline void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *ibdev) {} static inline int mlx5_ib_odp_init(void) { return 0; } static inline void mlx5_ib_odp_cleanup(void) {} @@ -1397,22 +1389,22 @@ int __mlx5_ib_add(struct mlx5_ib_dev *dev, const struct mlx5_ib_profile *profile); int mlx5_ib_get_vf_config(struct ib_device *device, int vf, - u8 port, struct ifla_vf_info *info); + u32 port, struct ifla_vf_info *info); int mlx5_ib_set_vf_link_state(struct ib_device *device, int vf, - u8 port, int state); + u32 port, int state); int mlx5_ib_get_vf_stats(struct ib_device *device, int vf, - u8 port, struct ifla_vf_stats *stats); -int mlx5_ib_get_vf_guid(struct ib_device *device, int vf, u8 port, + u32 port, struct ifla_vf_stats *stats); +int mlx5_ib_get_vf_guid(struct ib_device *device, int vf, u32 port, struct ifla_vf_guid *node_guid, struct ifla_vf_guid *port_guid); -int mlx5_ib_set_vf_guid(struct ib_device *device, int vf, u8 port, +int mlx5_ib_set_vf_guid(struct ib_device *device, int vf, u32 port, u64 guid, int type); __be16 mlx5_get_roce_udp_sport_min(const struct mlx5_ib_dev *dev, const struct ib_gid_attr *attr); -void mlx5_ib_cleanup_cong_debugfs(struct mlx5_ib_dev *dev, u8 port_num); -void mlx5_ib_init_cong_debugfs(struct mlx5_ib_dev *dev, u8 port_num); +void mlx5_ib_cleanup_cong_debugfs(struct mlx5_ib_dev *dev, u32 port_num); +void mlx5_ib_init_cong_debugfs(struct mlx5_ib_dev *dev, u32 port_num); /* GSI QP helper functions */ int mlx5_ib_create_gsi(struct ib_pd *pd, struct mlx5_ib_qp *mqp, @@ -1435,10 +1427,10 @@ void mlx5_ib_free_bfreg(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi, int bfregn); struct mlx5_ib_dev *mlx5_ib_get_ibdev_from_mpi(struct mlx5_ib_multiport_info *mpi); struct mlx5_core_dev *mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev *dev, - u8 ib_port_num, - u8 *native_port_num); + u32 ib_port_num, + u32 *native_port_num); void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *dev, - u8 port_num); + u32 port_num); extern const struct uapi_definition mlx5_ib_devx_defs[]; extern const struct uapi_definition mlx5_ib_flow_defs[]; diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index db05b0e0a8d7..4388afeff251 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -42,6 +42,7 @@ #include <rdma/ib_umem.h> #include <rdma/ib_umem_odp.h> #include <rdma/ib_verbs.h> +#include "dm.h" #include "mlx5_ib.h" /* @@ -119,8 +120,6 @@ mlx5_ib_create_mkey_cb(struct mlx5_ib_dev *dev, create_mkey_callback, context); } -static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr); -static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr); static int mr_cache_max_order(struct mlx5_ib_dev *dev); static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent); @@ -590,6 +589,8 @@ struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, ent->available_mrs--; queue_adjust_cache_locked(ent); spin_unlock_irq(&ent->lock); + + mlx5_clear_mr(mr); } mr->access_flags = access_flags; return mr; @@ -615,42 +616,20 @@ static struct mlx5_ib_mr *get_cache_mr(struct mlx5_cache_ent *req_ent) ent->available_mrs--; queue_adjust_cache_locked(ent); spin_unlock_irq(&ent->lock); - break; + mlx5_clear_mr(mr); + return mr; } queue_adjust_cache_locked(ent); spin_unlock_irq(&ent->lock); } - - if (!mr) - req_ent->miss++; - - return mr; -} - -static void detach_mr_from_cache(struct mlx5_ib_mr *mr) -{ - struct mlx5_cache_ent *ent = mr->cache_ent; - - mr->cache_ent = NULL; - spin_lock_irq(&ent->lock); - ent->total_mrs--; - spin_unlock_irq(&ent->lock); + req_ent->miss++; + return NULL; } -void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) +static void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) { struct mlx5_cache_ent *ent = mr->cache_ent; - if (!ent) - return; - - if (mlx5_mr_cache_invalidate(mr)) { - detach_mr_from_cache(mr); - destroy_mkey(dev, mr); - kfree(mr); - return; - } - spin_lock_irq(&ent->lock); list_add_tail(&mr->list, &ent->head); ent->available_mrs++; @@ -993,8 +972,6 @@ static struct mlx5_ib_mr *alloc_cacheable_mr(struct ib_pd *pd, mr->ibmr.pd = pd; mr->umem = umem; - mr->access_flags = access_flags; - mr->desc_size = sizeof(struct mlx5_mtt); mr->mmkey.iova = iova; mr->mmkey.size = umem->length; mr->mmkey.pd = to_mpd(pd)->pdn; @@ -1028,7 +1005,7 @@ static void *mlx5_ib_alloc_xlt(size_t *nents, size_t ent_size, gfp_t gfp_mask) */ might_sleep(); - gfp_mask |= __GFP_ZERO; + gfp_mask |= __GFP_ZERO | __GFP_NORETRY; /* * If the system already has a suitable high order page then just use @@ -1505,7 +1482,7 @@ static struct ib_mr *create_real_mr(struct ib_pd *pd, struct ib_umem *umem, */ err = mlx5_ib_update_mr_pas(mr, MLX5_IB_UPD_XLT_ENABLE); if (err) { - dereg_mr(dev, mr); + mlx5_ib_dereg_mr(&mr->ibmr, NULL); return ERR_PTR(err); } } @@ -1524,6 +1501,9 @@ static struct ib_mr *create_user_odp_mr(struct ib_pd *pd, u64 start, u64 length, if (!IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) return ERR_PTR(-EOPNOTSUPP); + err = mlx5r_odp_create_eq(dev, &dev->odp_pf_eq); + if (err) + return ERR_PTR(err); if (!start && length == U64_MAX) { if (iova != 0) return ERR_PTR(-EINVAL); @@ -1562,7 +1542,7 @@ static struct ib_mr *create_user_odp_mr(struct ib_pd *pd, u64 start, u64 length, return &mr->ibmr; err_dereg_mr: - dereg_mr(dev, mr); + mlx5_ib_dereg_mr(&mr->ibmr, NULL); return ERR_PTR(err); } @@ -1659,19 +1639,19 @@ struct ib_mr *mlx5_ib_reg_user_mr_dmabuf(struct ib_pd *pd, u64 offset, return &mr->ibmr; err_dereg_mr: - dereg_mr(dev, mr); + mlx5_ib_dereg_mr(&mr->ibmr, NULL); return ERR_PTR(err); } /** - * mlx5_mr_cache_invalidate - Fence all DMA on the MR + * revoke_mr - Fence all DMA on the MR * @mr: The MR to fence * * Upon return the NIC will not be doing any DMA to the pages under the MR, - * and any DMA inprogress will be completed. Failure of this function + * and any DMA in progress will be completed. Failure of this function * indicates the HW has failed catastrophically. */ -int mlx5_mr_cache_invalidate(struct mlx5_ib_mr *mr) +static int revoke_mr(struct mlx5_ib_mr *mr) { struct mlx5_umr_wr umrwr = {}; @@ -1765,7 +1745,7 @@ static int umr_rereg_pas(struct mlx5_ib_mr *mr, struct ib_pd *pd, * with it. This ensure the change is atomic relative to any use of the * MR. */ - err = mlx5_mr_cache_invalidate(mr); + err = revoke_mr(mr); if (err) return err; @@ -1844,7 +1824,7 @@ struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, * Only one active MR can refer to a umem at one time, revoke * the old MR before assigning the umem to the new one. */ - err = mlx5_mr_cache_invalidate(mr); + err = revoke_mr(mr); if (err) return ERR_PTR(err); umem = mr->umem; @@ -1931,7 +1911,7 @@ err: static void mlx5_free_priv_descs(struct mlx5_ib_mr *mr) { - if (mr->descs) { + if (!mr->umem && mr->descs) { struct ib_device *device = mr->ibmr.device; int size = mr->max_descs * mr->desc_size; struct mlx5_ib_dev *dev = to_mdev(device); @@ -1943,69 +1923,82 @@ mlx5_free_priv_descs(struct mlx5_ib_mr *mr) } } -static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) +int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) { - if (mr->sig) { + struct mlx5_ib_mr *mr = to_mmr(ibmr); + struct mlx5_ib_dev *dev = to_mdev(ibmr->device); + int rc; + + /* + * Any async use of the mr must hold the refcount, once the refcount + * goes to zero no other thread, such as ODP page faults, prefetch, any + * UMR activity, etc can touch the mkey. Thus it is safe to destroy it. + */ + if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) && + refcount_read(&mr->mmkey.usecount) != 0 && + xa_erase(&mr_to_mdev(mr)->odp_mkeys, mlx5_base_mkey(mr->mmkey.key))) + mlx5r_deref_wait_odp_mkey(&mr->mmkey); + + if (ibmr->type == IB_MR_TYPE_INTEGRITY) { + xa_cmpxchg(&dev->sig_mrs, mlx5_base_mkey(mr->mmkey.key), ibmr, + NULL, GFP_KERNEL); + + if (mr->mtt_mr) { + rc = mlx5_ib_dereg_mr(&mr->mtt_mr->ibmr, NULL); + if (rc) + return rc; + mr->mtt_mr = NULL; + } + if (mr->klm_mr) { + rc = mlx5_ib_dereg_mr(&mr->klm_mr->ibmr, NULL); + if (rc) + return rc; + mr->klm_mr = NULL; + } + if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_memory.psv_idx)) mlx5_ib_warn(dev, "failed to destroy mem psv %d\n", mr->sig->psv_memory.psv_idx); - if (mlx5_core_destroy_psv(dev->mdev, - mr->sig->psv_wire.psv_idx)) + if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_wire.psv_idx)) mlx5_ib_warn(dev, "failed to destroy wire psv %d\n", mr->sig->psv_wire.psv_idx); - xa_erase(&dev->sig_mrs, mlx5_base_mkey(mr->mmkey.key)); kfree(mr->sig); mr->sig = NULL; } + /* Stop DMA */ + if (mr->cache_ent) { + if (revoke_mr(mr)) { + spin_lock_irq(&mr->cache_ent->lock); + mr->cache_ent->total_mrs--; + spin_unlock_irq(&mr->cache_ent->lock); + mr->cache_ent = NULL; + } + } if (!mr->cache_ent) { - destroy_mkey(dev, mr); - mlx5_free_priv_descs(mr); + rc = destroy_mkey(to_mdev(mr->ibmr.device), mr); + if (rc) + return rc; } -} -static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) -{ - struct ib_umem *umem = mr->umem; + if (mr->umem) { + bool is_odp = is_odp_mr(mr); - /* Stop all DMA */ - if (is_odp_mr(mr)) - mlx5_ib_fence_odp_mr(mr); - else if (is_dmabuf_mr(mr)) - mlx5_ib_fence_dmabuf_mr(mr); - else - clean_mr(dev, mr); - - if (umem) { - if (!is_odp_mr(mr)) - atomic_sub(ib_umem_num_pages(umem), + if (!is_odp) + atomic_sub(ib_umem_num_pages(mr->umem), &dev->mdev->priv.reg_pages); - ib_umem_release(umem); + ib_umem_release(mr->umem); + if (is_odp) + mlx5_ib_free_odp_mr(mr); } - if (mr->cache_ent) + if (mr->cache_ent) { mlx5_mr_cache_free(dev, mr); - else + } else { + mlx5_free_priv_descs(mr); kfree(mr); -} - -int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) -{ - struct mlx5_ib_mr *mmr = to_mmr(ibmr); - - if (ibmr->type == IB_MR_TYPE_INTEGRITY) { - dereg_mr(to_mdev(mmr->mtt_mr->ibmr.device), mmr->mtt_mr); - dereg_mr(to_mdev(mmr->klm_mr->ibmr.device), mmr->klm_mr); } - - if (is_odp_mr(mmr) && to_ib_umem_odp(mmr->umem)->is_implicit_odp) { - mlx5_ib_free_implicit_mr(mmr); - return 0; - } - - dereg_mr(to_mdev(ibmr->device), mmr); - return 0; } @@ -2177,10 +2170,10 @@ err_free_descs: destroy_mkey(dev, mr); mlx5_free_priv_descs(mr); err_free_mtt_mr: - dereg_mr(to_mdev(mr->mtt_mr->ibmr.device), mr->mtt_mr); + mlx5_ib_dereg_mr(&mr->mtt_mr->ibmr, NULL); mr->mtt_mr = NULL; err_free_klm_mr: - dereg_mr(to_mdev(mr->klm_mr->ibmr.device), mr->klm_mr); + mlx5_ib_dereg_mr(&mr->klm_mr->ibmr, NULL); mr->klm_mr = NULL; err_destroy_psv: if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_memory.psv_idx)) diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index b103555b1f5d..782b2af8f211 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -181,64 +181,29 @@ void mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries, } } -static void dma_fence_odp_mr(struct mlx5_ib_mr *mr) -{ - struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); - - /* Ensure mlx5_ib_invalidate_range() will not touch the MR any more */ - mutex_lock(&odp->umem_mutex); - if (odp->npages) { - mlx5_mr_cache_invalidate(mr); - ib_umem_odp_unmap_dma_pages(odp, ib_umem_start(odp), - ib_umem_end(odp)); - WARN_ON(odp->npages); - } - odp->private = NULL; - mutex_unlock(&odp->umem_mutex); - - if (!mr->cache_ent) { - mlx5_core_destroy_mkey(mr_to_mdev(mr)->mdev, &mr->mmkey); - WARN_ON(mr->descs); - } -} - /* * This must be called after the mr has been removed from implicit_children. * NOTE: The MR does not necessarily have to be * empty here, parallel page faults could have raced with the free process and * added pages to it. */ -static void free_implicit_child_mr(struct mlx5_ib_mr *mr, bool need_imr_xlt) +static void free_implicit_child_mr_work(struct work_struct *work) { + struct mlx5_ib_mr *mr = + container_of(work, struct mlx5_ib_mr, odp_destroy.work); struct mlx5_ib_mr *imr = mr->parent; struct ib_umem_odp *odp_imr = to_ib_umem_odp(imr->umem); struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); - unsigned long idx = ib_umem_start(odp) >> MLX5_IMR_MTT_SHIFT; mlx5r_deref_wait_odp_mkey(&mr->mmkey); - if (need_imr_xlt) { - mutex_lock(&odp_imr->umem_mutex); - mlx5_ib_update_xlt(mr->parent, idx, 1, 0, - MLX5_IB_UPD_XLT_INDIRECT | - MLX5_IB_UPD_XLT_ATOMIC); - mutex_unlock(&odp_imr->umem_mutex); - } - - dma_fence_odp_mr(mr); - - mr->parent = NULL; - mlx5_mr_cache_free(mr_to_mdev(mr), mr); - ib_umem_odp_release(odp); -} - -static void free_implicit_child_mr_work(struct work_struct *work) -{ - struct mlx5_ib_mr *mr = - container_of(work, struct mlx5_ib_mr, odp_destroy.work); - struct mlx5_ib_mr *imr = mr->parent; + mutex_lock(&odp_imr->umem_mutex); + mlx5_ib_update_xlt(mr->parent, ib_umem_start(odp) >> MLX5_IMR_MTT_SHIFT, + 1, 0, + MLX5_IB_UPD_XLT_INDIRECT | MLX5_IB_UPD_XLT_ATOMIC); + mutex_unlock(&odp_imr->umem_mutex); + mlx5_ib_dereg_mr(&mr->ibmr, NULL); - free_implicit_child_mr(mr, true); mlx5r_deref_odp_mkey(&imr->mmkey); } @@ -352,7 +317,7 @@ const struct mmu_interval_notifier_ops mlx5_mn_ops = { .invalidate = mlx5_ib_invalidate_range, }; -void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev) +static void internal_fill_odp_caps(struct mlx5_ib_dev *dev) { struct ib_odp_caps *caps = &dev->odp_caps; @@ -455,8 +420,10 @@ static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr, ret = mr = mlx5_mr_cache_alloc( mr_to_mdev(imr), MLX5_IMR_MTT_CACHE_ENTRY, imr->access_flags); - if (IS_ERR(mr)) - goto out_umem; + if (IS_ERR(mr)) { + ib_umem_odp_release(odp); + return mr; + } mr->ibmr.pd = imr->ibmr.pd; mr->ibmr.device = &mr_to_mdev(imr)->ib_dev; @@ -506,9 +473,7 @@ static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr, out_lock: xa_unlock(&imr->implicit_children); out_mr: - mlx5_mr_cache_free(mr_to_mdev(imr), mr); -out_umem: - ib_umem_odp_release(odp); + mlx5_ib_dereg_mr(&mr->ibmr, NULL); return ret; } @@ -531,8 +496,8 @@ struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd, imr = mlx5_mr_cache_alloc(dev, MLX5_IMR_KSM_CACHE_ENTRY, access_flags); if (IS_ERR(imr)) { - err = PTR_ERR(imr); - goto out_umem; + ib_umem_odp_release(umem_odp); + return imr; } imr->ibmr.pd = &pd->ibpd; @@ -562,93 +527,22 @@ struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd, return imr; out_mr: mlx5_ib_err(dev, "Failed to register MKEY %d\n", err); - mlx5_mr_cache_free(dev, imr); -out_umem: - ib_umem_odp_release(umem_odp); + mlx5_ib_dereg_mr(&imr->ibmr, NULL); return ERR_PTR(err); } -void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *imr) +void mlx5_ib_free_odp_mr(struct mlx5_ib_mr *mr) { - struct ib_umem_odp *odp_imr = to_ib_umem_odp(imr->umem); - struct mlx5_ib_dev *dev = mr_to_mdev(imr); struct mlx5_ib_mr *mtt; unsigned long idx; - xa_erase(&dev->odp_mkeys, mlx5_base_mkey(imr->mmkey.key)); - /* - * All work on the prefetch list must be completed, xa_erase() prevented - * new work from being created. - */ - mlx5r_deref_wait_odp_mkey(&imr->mmkey); - /* - * At this point it is forbidden for any other thread to enter - * pagefault_mr() on this imr. It is already forbidden to call - * pagefault_mr() on an implicit child. Due to this additions to - * implicit_children are prevented. - * In addition, any new call to destroy_unused_implicit_child_mr() - * may return immediately. - */ - /* - * Fence the imr before we destroy the children. This allows us to - * skip updating the XLT of the imr during destroy of the child mkey - * the imr points to. + * If this is an implicit MR it is already invalidated so we can just + * delete the children mkeys. */ - mlx5_mr_cache_invalidate(imr); - - xa_for_each(&imr->implicit_children, idx, mtt) { - xa_erase(&imr->implicit_children, idx); - free_implicit_child_mr(mtt, false); - } - - mlx5_mr_cache_free(dev, imr); - ib_umem_odp_release(odp_imr); -} - -/** - * mlx5_ib_fence_odp_mr - Stop all access to the ODP MR - * @mr: to fence - * - * On return no parallel threads will be touching this MR and no DMA will be - * active. - */ -void mlx5_ib_fence_odp_mr(struct mlx5_ib_mr *mr) -{ - /* Prevent new page faults and prefetch requests from succeeding */ - xa_erase(&mr_to_mdev(mr)->odp_mkeys, mlx5_base_mkey(mr->mmkey.key)); - - /* Wait for all running page-fault handlers to finish. */ - mlx5r_deref_wait_odp_mkey(&mr->mmkey); - - dma_fence_odp_mr(mr); -} - -/** - * mlx5_ib_fence_dmabuf_mr - Stop all access to the dmabuf MR - * @mr: to fence - * - * On return no parallel threads will be touching this MR and no DMA will be - * active. - */ -void mlx5_ib_fence_dmabuf_mr(struct mlx5_ib_mr *mr) -{ - struct ib_umem_dmabuf *umem_dmabuf = to_ib_umem_dmabuf(mr->umem); - - /* Prevent new page faults and prefetch requests from succeeding */ - xa_erase(&mr_to_mdev(mr)->odp_mkeys, mlx5_base_mkey(mr->mmkey.key)); - - mlx5r_deref_wait_odp_mkey(&mr->mmkey); - - dma_resv_lock(umem_dmabuf->attach->dmabuf->resv, NULL); - mlx5_mr_cache_invalidate(mr); - umem_dmabuf->private = NULL; - ib_umem_dmabuf_unmap_pages(umem_dmabuf); - dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv); - - if (!mr->cache_ent) { - mlx5_core_destroy_mkey(mr_to_mdev(mr)->mdev, &mr->mmkey); - WARN_ON(mr->descs); + xa_for_each(&mr->implicit_children, idx, mtt) { + xa_erase(&mr->implicit_children, idx); + mlx5_ib_dereg_mr(&mtt->ibmr, NULL); } } @@ -1637,20 +1531,24 @@ enum { MLX5_IB_NUM_PF_DRAIN = 64, }; -static int -mlx5_ib_create_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) +int mlx5r_odp_create_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) { struct mlx5_eq_param param = {}; - int err; + int err = 0; + mutex_lock(&dev->odp_eq_mutex); + if (eq->core) + goto unlock; INIT_WORK(&eq->work, mlx5_ib_eq_pf_action); spin_lock_init(&eq->lock); eq->dev = dev; eq->pool = mempool_create_kmalloc_pool(MLX5_IB_NUM_PF_DRAIN, sizeof(struct mlx5_pagefault)); - if (!eq->pool) - return -ENOMEM; + if (!eq->pool) { + err = -ENOMEM; + goto unlock; + } eq->wq = alloc_workqueue("mlx5_ib_page_fault", WQ_HIGHPRI | WQ_UNBOUND | WQ_MEM_RECLAIM, @@ -1661,7 +1559,7 @@ mlx5_ib_create_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) } eq->irq_nb.notifier_call = mlx5_ib_eq_pf_int; - param = (struct mlx5_eq_param) { + param = (struct mlx5_eq_param){ .irq_index = 0, .nent = MLX5_IB_NUM_PF_EQE, }; @@ -1677,21 +1575,27 @@ mlx5_ib_create_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) goto err_eq; } + mutex_unlock(&dev->odp_eq_mutex); return 0; err_eq: mlx5_eq_destroy_generic(dev->mdev, eq->core); err_wq: + eq->core = NULL; destroy_workqueue(eq->wq); err_mempool: mempool_destroy(eq->pool); +unlock: + mutex_unlock(&dev->odp_eq_mutex); return err; } static int -mlx5_ib_destroy_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) +mlx5_ib_odp_destroy_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) { int err; + if (!eq->core) + return 0; mlx5_eq_disable(dev->mdev, eq->core, &eq->irq_nb); err = mlx5_eq_destroy_generic(dev->mdev, eq->core); cancel_work_sync(&eq->work); @@ -1735,6 +1639,8 @@ int mlx5_ib_odp_init_one(struct mlx5_ib_dev *dev) { int ret = 0; + internal_fill_odp_caps(dev); + if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT)) return ret; @@ -1748,8 +1654,7 @@ int mlx5_ib_odp_init_one(struct mlx5_ib_dev *dev) } } - ret = mlx5_ib_create_pf_eq(dev, &dev->odp_pf_eq); - + mutex_init(&dev->odp_eq_mutex); return ret; } @@ -1758,7 +1663,7 @@ void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *dev) if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT)) return; - mlx5_ib_destroy_pf_eq(dev, &dev->odp_pf_eq); + mlx5_ib_odp_destroy_eq(dev, &dev->odp_pf_eq); } int mlx5_ib_odp_init(void) diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index f5a52a6fae43..9282eb10bfae 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -67,7 +67,7 @@ struct mlx5_modify_raw_qp_param { struct mlx5_rate_limit rl; u8 rq_q_ctr_id; - u16 port; + u32 port; }; static void get_cqs(enum ib_qp_type qp_type, @@ -3146,6 +3146,19 @@ enum { MLX5_PATH_FLAG_COUNTER = 1 << 2, }; +static int mlx5_to_ib_rate_map(u8 rate) +{ + static const int rates[] = { IB_RATE_PORT_CURRENT, IB_RATE_56_GBPS, + IB_RATE_25_GBPS, IB_RATE_100_GBPS, + IB_RATE_200_GBPS, IB_RATE_50_GBPS, + IB_RATE_400_GBPS }; + + if (rate < ARRAY_SIZE(rates)) + return rates[rate]; + + return rate - MLX5_STAT_RATE_OFFSET; +} + static int ib_to_mlx5_rate_map(u8 rate) { switch (rate) { @@ -4485,7 +4498,7 @@ static void to_rdma_ah_attr(struct mlx5_ib_dev *ibdev, rdma_ah_set_path_bits(ah_attr, MLX5_GET(ads, path, mlid)); static_rate = MLX5_GET(ads, path, stat_rate); - rdma_ah_set_static_rate(ah_attr, static_rate ? static_rate - 5 : 0); + rdma_ah_set_static_rate(ah_attr, mlx5_to_ib_rate_map(static_rate)); if (MLX5_GET(ads, path, grh) || ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) { rdma_ah_set_grh(ah_attr, NULL, MLX5_GET(ads, path, flow_label), diff --git a/drivers/infiniband/hw/mlx5/std_types.c b/drivers/infiniband/hw/mlx5/std_types.c index 16145fda68d0..c0ddf7b3c6e2 100644 --- a/drivers/infiniband/hw/mlx5/std_types.c +++ b/drivers/infiniband/hw/mlx5/std_types.c @@ -7,6 +7,8 @@ #include <rdma/mlx5_user_ioctl_cmds.h> #include <rdma/mlx5_user_ioctl_verbs.h> #include <linux/mlx5/driver.h> +#include <linux/mlx5/eswitch.h> +#include <linux/mlx5/vport.h> #include "mlx5_ib.h" #define UVERBS_MODULE_NAME mlx5_ib @@ -23,6 +25,174 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_PD_QUERY)( &mpd->pdn, sizeof(mpd->pdn)); } +static int fill_vport_icm_addr(struct mlx5_core_dev *mdev, u16 vport, + struct mlx5_ib_uapi_query_port *info) +{ + u32 out[MLX5_ST_SZ_DW(query_esw_vport_context_out)] = {}; + u32 in[MLX5_ST_SZ_DW(query_esw_vport_context_in)] = {}; + bool sw_owner_supp; + u64 icm_rx; + u64 icm_tx; + int err; + + sw_owner_supp = MLX5_CAP_ESW_FLOWTABLE_FDB(mdev, sw_owner) || + MLX5_CAP_ESW_FLOWTABLE_FDB(mdev, sw_owner_v2); + + if (vport == MLX5_VPORT_UPLINK) { + icm_rx = MLX5_CAP64_ESW_FLOWTABLE(mdev, + sw_steering_uplink_icm_address_rx); + icm_tx = MLX5_CAP64_ESW_FLOWTABLE(mdev, + sw_steering_uplink_icm_address_tx); + } else { + MLX5_SET(query_esw_vport_context_in, in, opcode, + MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT); + MLX5_SET(query_esw_vport_context_in, in, vport_number, vport); + MLX5_SET(query_esw_vport_context_in, in, other_vport, true); + + err = mlx5_cmd_exec_inout(mdev, query_esw_vport_context, in, + out); + + if (err) + return err; + + icm_rx = MLX5_GET64( + query_esw_vport_context_out, out, + esw_vport_context.sw_steering_vport_icm_address_rx); + + icm_tx = MLX5_GET64( + query_esw_vport_context_out, out, + esw_vport_context.sw_steering_vport_icm_address_tx); + } + + if (sw_owner_supp && icm_rx) { + info->vport_steering_icm_rx = icm_rx; + info->flags |= + MLX5_IB_UAPI_QUERY_PORT_VPORT_STEERING_ICM_RX; + } + + if (sw_owner_supp && icm_tx) { + info->vport_steering_icm_tx = icm_tx; + info->flags |= + MLX5_IB_UAPI_QUERY_PORT_VPORT_STEERING_ICM_TX; + } + + return 0; +} + +static int fill_vport_vhca_id(struct mlx5_core_dev *mdev, u16 vport, + struct mlx5_ib_uapi_query_port *info) +{ + size_t out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); + u32 in[MLX5_ST_SZ_DW(query_hca_cap_in)] = {}; + void *out; + int err; + + out = kzalloc(out_sz, GFP_KERNEL); + if (!out) + return -ENOMEM; + + MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); + MLX5_SET(query_hca_cap_in, in, other_function, true); + MLX5_SET(query_hca_cap_in, in, function_id, vport); + MLX5_SET(query_hca_cap_in, in, op_mod, + MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE | + HCA_CAP_OPMOD_GET_CUR); + + err = mlx5_cmd_exec(mdev, in, sizeof(in), out, out_sz); + if (err) + goto out; + + info->vport_vhca_id = MLX5_GET(query_hca_cap_out, out, + capability.cmd_hca_cap.vhca_id); + + info->flags |= MLX5_IB_UAPI_QUERY_PORT_VPORT_VHCA_ID; +out: + kfree(out); + return err; +} + +static int fill_switchdev_info(struct mlx5_ib_dev *dev, u32 port_num, + struct mlx5_ib_uapi_query_port *info) +{ + struct mlx5_core_dev *mdev = dev->mdev; + struct mlx5_eswitch_rep *rep; + int err; + + rep = dev->port[port_num - 1].rep; + if (!rep) + return -EOPNOTSUPP; + + info->vport = rep->vport; + info->flags |= MLX5_IB_UAPI_QUERY_PORT_VPORT; + + if (rep->vport != MLX5_VPORT_UPLINK) { + err = fill_vport_vhca_id(mdev, rep->vport, info); + if (err) + return err; + } + + info->esw_owner_vhca_id = MLX5_CAP_GEN(mdev, vhca_id); + info->flags |= MLX5_IB_UAPI_QUERY_PORT_ESW_OWNER_VHCA_ID; + + err = fill_vport_icm_addr(mdev, rep->vport, info); + if (err) + return err; + + if (mlx5_eswitch_vport_match_metadata_enabled(mdev->priv.eswitch)) { + info->reg_c0.value = mlx5_eswitch_get_vport_metadata_for_match( + mdev->priv.eswitch, rep->vport); + info->reg_c0.mask = mlx5_eswitch_get_vport_metadata_mask(); + info->flags |= MLX5_IB_UAPI_QUERY_PORT_VPORT_REG_C0; + } + + return 0; +} + +static int UVERBS_HANDLER(MLX5_IB_METHOD_QUERY_PORT)( + struct uverbs_attr_bundle *attrs) +{ + struct mlx5_ib_uapi_query_port info = {}; + struct mlx5_ib_ucontext *c; + struct mlx5_ib_dev *dev; + u32 port_num; + int ret; + + if (uverbs_copy_from(&port_num, attrs, + MLX5_IB_ATTR_QUERY_PORT_PORT_NUM)) + return -EFAULT; + + c = to_mucontext(ib_uverbs_get_ucontext(attrs)); + if (IS_ERR(c)) + return PTR_ERR(c); + dev = to_mdev(c->ibucontext.device); + + if (!rdma_is_port_valid(&dev->ib_dev, port_num)) + return -EINVAL; + + if (mlx5_eswitch_mode(dev->mdev) == MLX5_ESWITCH_OFFLOADS) { + ret = fill_switchdev_info(dev, port_num, &info); + if (ret) + return ret; + } + + return uverbs_copy_to_struct_or_zero(attrs, MLX5_IB_ATTR_QUERY_PORT, &info, + sizeof(info)); +} + +DECLARE_UVERBS_NAMED_METHOD( + MLX5_IB_METHOD_QUERY_PORT, + UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_QUERY_PORT_PORT_NUM, + UVERBS_ATTR_TYPE(u32), UA_MANDATORY), + UVERBS_ATTR_PTR_OUT( + MLX5_IB_ATTR_QUERY_PORT, + UVERBS_ATTR_STRUCT(struct mlx5_ib_uapi_query_port, + reg_c0), + UA_MANDATORY)); + +ADD_UVERBS_METHODS(mlx5_ib_device, + UVERBS_OBJECT_DEVICE, + &UVERBS_METHOD(MLX5_IB_METHOD_QUERY_PORT)); + DECLARE_UVERBS_NAMED_METHOD( MLX5_IB_METHOD_PD_QUERY, UVERBS_ATTR_IDR(MLX5_IB_ATTR_QUERY_PD_HANDLE, @@ -41,5 +211,8 @@ const struct uapi_definition mlx5_ib_std_types_defs[] = { UAPI_DEF_CHAIN_OBJ_TREE( UVERBS_OBJECT_PD, &mlx5_ib_pd), + UAPI_DEF_CHAIN_OBJ_TREE( + UVERBS_OBJECT_DEVICE, + &mlx5_ib_device), {}, }; diff --git a/drivers/infiniband/hw/mthca/mthca_av.c b/drivers/infiniband/hw/mthca/mthca_av.c index f051f4e06b53..3df1f5ff7932 100644 --- a/drivers/infiniband/hw/mthca/mthca_av.c +++ b/drivers/infiniband/hw/mthca/mthca_av.c @@ -91,7 +91,7 @@ static enum ib_rate tavor_rate_to_ib(u8 mthca_rate, u8 port_rate) } } -enum ib_rate mthca_rate_to_ib(struct mthca_dev *dev, u8 mthca_rate, u8 port) +enum ib_rate mthca_rate_to_ib(struct mthca_dev *dev, u8 mthca_rate, u32 port) { if (mthca_is_memfree(dev)) { /* Handle old Arbel FW */ @@ -131,7 +131,7 @@ static u8 ib_rate_to_tavor(u8 static_rate) } } -u8 mthca_get_rate(struct mthca_dev *dev, int static_rate, u8 port) +u8 mthca_get_rate(struct mthca_dev *dev, int static_rate, u32 port) { u8 rate; @@ -293,7 +293,7 @@ int mthca_ah_query(struct ib_ah *ibah, struct rdma_ah_attr *attr) { struct mthca_ah *ah = to_mah(ibah); struct mthca_dev *dev = to_mdev(ibah->device); - u8 port_num = be32_to_cpu(ah->av->port_pd) >> 24; + u32 port_num = be32_to_cpu(ah->av->port_pd) >> 24; /* Only implement for MAD and memfree ah for now. */ if (ah->type == MTHCA_AH_ON_HCA) diff --git a/drivers/infiniband/hw/mthca/mthca_dev.h b/drivers/infiniband/hw/mthca/mthca_dev.h index a445160de3e1..a4a9d871d00e 100644 --- a/drivers/infiniband/hw/mthca/mthca_dev.h +++ b/drivers/infiniband/hw/mthca/mthca_dev.h @@ -546,7 +546,7 @@ int mthca_alloc_sqp(struct mthca_dev *dev, enum ib_sig_type send_policy, struct ib_qp_cap *cap, int qpn, - int port, + u32 port, struct mthca_qp *qp, struct ib_udata *udata); void mthca_free_qp(struct mthca_dev *dev, struct mthca_qp *qp); @@ -559,13 +559,13 @@ int mthca_read_ah(struct mthca_dev *dev, struct mthca_ah *ah, struct ib_ud_header *header); int mthca_ah_query(struct ib_ah *ibah, struct rdma_ah_attr *attr); int mthca_ah_grh_present(struct mthca_ah *ah); -u8 mthca_get_rate(struct mthca_dev *dev, int static_rate, u8 port); -enum ib_rate mthca_rate_to_ib(struct mthca_dev *dev, u8 mthca_rate, u8 port); +u8 mthca_get_rate(struct mthca_dev *dev, int static_rate, u32 port); +enum ib_rate mthca_rate_to_ib(struct mthca_dev *dev, u8 mthca_rate, u32 port); int mthca_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid); int mthca_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid); -int mthca_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, +int mthca_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const struct ib_mad *in, struct ib_mad *out, size_t *out_mad_size, u16 *out_mad_pkey_index); diff --git a/drivers/infiniband/hw/mthca/mthca_mad.c b/drivers/infiniband/hw/mthca/mthca_mad.c index 99aa8183a7f2..04252700790e 100644 --- a/drivers/infiniband/hw/mthca/mthca_mad.c +++ b/drivers/infiniband/hw/mthca/mthca_mad.c @@ -162,7 +162,7 @@ static void node_desc_override(struct ib_device *dev, } static void forward_trap(struct mthca_dev *dev, - u8 port_num, + u32 port_num, const struct ib_mad *mad) { int qpn = mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_SUBN_LID_ROUTED; @@ -196,7 +196,7 @@ static void forward_trap(struct mthca_dev *dev, } } -int mthca_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, +int mthca_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const struct ib_mad *in, struct ib_mad *out, size_t *out_mad_size, u16 *out_mad_pkey_index) diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c index 1a3dd07f993b..522bb606120e 100644 --- a/drivers/infiniband/hw/mthca/mthca_provider.c +++ b/drivers/infiniband/hw/mthca/mthca_provider.c @@ -127,7 +127,7 @@ static int mthca_query_device(struct ib_device *ibdev, struct ib_device_attr *pr } static int mthca_query_port(struct ib_device *ibdev, - u8 port, struct ib_port_attr *props) + u32 port, struct ib_port_attr *props) { struct ib_smp *in_mad = NULL; struct ib_smp *out_mad = NULL; @@ -194,7 +194,7 @@ static int mthca_modify_device(struct ib_device *ibdev, } static int mthca_modify_port(struct ib_device *ibdev, - u8 port, int port_modify_mask, + u32 port, int port_modify_mask, struct ib_port_modify *props) { struct mthca_set_ib_param set_ib; @@ -223,7 +223,7 @@ out: } static int mthca_query_pkey(struct ib_device *ibdev, - u8 port, u16 index, u16 *pkey) + u32 port, u16 index, u16 *pkey) { struct ib_smp *in_mad = NULL; struct ib_smp *out_mad = NULL; @@ -251,7 +251,7 @@ static int mthca_query_pkey(struct ib_device *ibdev, return err; } -static int mthca_query_gid(struct ib_device *ibdev, u8 port, +static int mthca_query_gid(struct ib_device *ibdev, u32 port, int index, union ib_gid *gid) { struct ib_smp *in_mad = NULL; @@ -1051,7 +1051,7 @@ out: return err; } -static int mthca_port_immutable(struct ib_device *ibdev, u8 port_num, +static int mthca_port_immutable(struct ib_device *ibdev, u32 port_num, struct ib_port_immutable *immutable) { struct ib_port_attr attr; diff --git a/drivers/infiniband/hw/mthca/mthca_qp.c b/drivers/infiniband/hw/mthca/mthca_qp.c index 07cfc0934b17..69bba0ef4a5d 100644 --- a/drivers/infiniband/hw/mthca/mthca_qp.c +++ b/drivers/infiniband/hw/mthca/mthca_qp.c @@ -1370,7 +1370,7 @@ int mthca_alloc_sqp(struct mthca_dev *dev, enum ib_sig_type send_policy, struct ib_qp_cap *cap, int qpn, - int port, + u32 port, struct mthca_qp *qp, struct ib_udata *udata) { diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_ah.c b/drivers/infiniband/hw/ocrdma/ocrdma_ah.c index 699a8b719ed6..88c45928301f 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_ah.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_ah.c @@ -250,7 +250,7 @@ int ocrdma_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr) } int ocrdma_process_mad(struct ib_device *ibdev, int process_mad_flags, - u8 port_num, const struct ib_wc *in_wc, + u32 port_num, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const struct ib_mad *in, struct ib_mad *out, size_t *out_mad_size, u16 *out_mad_pkey_index) diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_ah.h b/drivers/infiniband/hw/ocrdma/ocrdma_ah.h index 35cf2e2ff391..2626679df31d 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_ah.h +++ b/drivers/infiniband/hw/ocrdma/ocrdma_ah.h @@ -57,7 +57,7 @@ int ocrdma_destroy_ah(struct ib_ah *ah, u32 flags); int ocrdma_query_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr); int ocrdma_process_mad(struct ib_device *dev, int process_mad_flags, - u8 port_num, const struct ib_wc *in_wc, + u32 port_num, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const struct ib_mad *in, struct ib_mad *out, size_t *out_mad_size, u16 *out_mad_pkey_index); diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_main.c b/drivers/infiniband/hw/ocrdma/ocrdma_main.c index 9a834a9cca0e..4882b3156edb 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_main.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_main.c @@ -77,12 +77,12 @@ void ocrdma_get_guid(struct ocrdma_dev *dev, u8 *guid) guid[7] = mac_addr[5]; } static enum rdma_link_layer ocrdma_link_layer(struct ib_device *device, - u8 port_num) + u32 port_num) { return IB_LINK_LAYER_ETHERNET; } -static int ocrdma_port_immutable(struct ib_device *ibdev, u8 port_num, +static int ocrdma_port_immutable(struct ib_device *ibdev, u32 port_num, struct ib_port_immutable *immutable) { struct ib_port_attr attr; diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c index 3acb5c10b155..58619ce64d0d 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c @@ -54,7 +54,7 @@ #include "ocrdma_verbs.h" #include <rdma/ocrdma-abi.h> -int ocrdma_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) +int ocrdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) { if (index > 0) return -EINVAL; @@ -150,7 +150,7 @@ static inline void get_link_speed_and_width(struct ocrdma_dev *dev, } int ocrdma_query_port(struct ib_device *ibdev, - u8 port, struct ib_port_attr *props) + u32 port, struct ib_port_attr *props) { enum ib_port_state port_state; struct ocrdma_dev *dev; diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h index 425d554e7f3f..b1c5fad81603 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h @@ -53,13 +53,14 @@ int ocrdma_arm_cq(struct ib_cq *, enum ib_cq_notify_flags flags); int ocrdma_query_device(struct ib_device *, struct ib_device_attr *props, struct ib_udata *uhw); -int ocrdma_query_port(struct ib_device *, u8 port, struct ib_port_attr *props); +int ocrdma_query_port(struct ib_device *ibdev, u32 port, + struct ib_port_attr *props); enum rdma_protocol_type -ocrdma_query_protocol(struct ib_device *device, u8 port_num); +ocrdma_query_protocol(struct ib_device *device, u32 port_num); void ocrdma_get_guid(struct ocrdma_dev *, u8 *guid); -int ocrdma_query_pkey(struct ib_device *, u8 port, u16 index, u16 *pkey); +int ocrdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey); int ocrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata); void ocrdma_dealloc_ucontext(struct ib_ucontext *uctx); diff --git a/drivers/infiniband/hw/qedr/main.c b/drivers/infiniband/hw/qedr/main.c index 8e7c069e1a2d..8334a9850220 100644 --- a/drivers/infiniband/hw/qedr/main.c +++ b/drivers/infiniband/hw/qedr/main.c @@ -53,7 +53,7 @@ MODULE_LICENSE("Dual BSD/GPL"); #define QEDR_WQ_MULTIPLIER_DFT (3) -static void qedr_ib_dispatch_event(struct qedr_dev *dev, u8 port_num, +static void qedr_ib_dispatch_event(struct qedr_dev *dev, u32 port_num, enum ib_event_type type) { struct ib_event ibev; @@ -66,7 +66,7 @@ static void qedr_ib_dispatch_event(struct qedr_dev *dev, u8 port_num, } static enum rdma_link_layer qedr_link_layer(struct ib_device *device, - u8 port_num) + u32 port_num) { return IB_LINK_LAYER_ETHERNET; } @@ -81,7 +81,7 @@ static void qedr_get_dev_fw_str(struct ib_device *ibdev, char *str) (fw_ver >> 8) & 0xFF, fw_ver & 0xFF); } -static int qedr_roce_port_immutable(struct ib_device *ibdev, u8 port_num, +static int qedr_roce_port_immutable(struct ib_device *ibdev, u32 port_num, struct ib_port_immutable *immutable) { struct ib_port_attr attr; @@ -100,7 +100,7 @@ static int qedr_roce_port_immutable(struct ib_device *ibdev, u8 port_num, return 0; } -static int qedr_iw_port_immutable(struct ib_device *ibdev, u8 port_num, +static int qedr_iw_port_immutable(struct ib_device *ibdev, u32 port_num, struct ib_port_immutable *immutable) { struct ib_port_attr attr; diff --git a/drivers/infiniband/hw/qedr/qedr_iw_cm.c b/drivers/infiniband/hw/qedr/qedr_iw_cm.c index c4bc58736e48..1715fbe0719d 100644 --- a/drivers/infiniband/hw/qedr/qedr_iw_cm.c +++ b/drivers/infiniband/hw/qedr/qedr_iw_cm.c @@ -636,8 +636,10 @@ int qedr_iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) memcpy(in_params.local_mac_addr, dev->ndev->dev_addr, ETH_ALEN); if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT, - &qp->iwarp_cm_flags)) + &qp->iwarp_cm_flags)) { + rc = -ENODEV; goto err; /* QP already being destroyed */ + } rc = dev->ops->iwarp_connect(dev->rdma_ctx, &in_params, &out_params); if (rc) { diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c index 9ea542270ed4..fdc47ef7d861 100644 --- a/drivers/infiniband/hw/qedr/verbs.c +++ b/drivers/infiniband/hw/qedr/verbs.c @@ -72,7 +72,7 @@ static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src, return ib_copy_to_udata(udata, src, min_len); } -int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) +int qedr_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) { if (index >= QEDR_ROCE_PKEY_TABLE_LEN) return -EINVAL; @@ -81,7 +81,7 @@ int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) return 0; } -int qedr_iw_query_gid(struct ib_device *ibdev, u8 port, +int qedr_iw_query_gid(struct ib_device *ibdev, u32 port, int index, union ib_gid *sgid) { struct qedr_dev *dev = get_qedr_dev(ibdev); @@ -210,7 +210,8 @@ static inline void get_link_speed_and_width(int speed, u16 *ib_speed, } } -int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr) +int qedr_query_port(struct ib_device *ibdev, u32 port, + struct ib_port_attr *attr) { struct qedr_dev *dev; struct qed_rdma_port *rdma_port; @@ -4483,7 +4484,7 @@ int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) } int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags, - u8 port_num, const struct ib_wc *in_wc, + u32 port_num, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const struct ib_mad *in, struct ib_mad *out_mad, size_t *out_mad_size, u16 *out_mad_pkey_index) diff --git a/drivers/infiniband/hw/qedr/verbs.h b/drivers/infiniband/hw/qedr/verbs.h index 2672c32bc2f7..34ad47515861 100644 --- a/drivers/infiniband/hw/qedr/verbs.h +++ b/drivers/infiniband/hw/qedr/verbs.h @@ -34,12 +34,13 @@ int qedr_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, struct ib_udata *udata); -int qedr_query_port(struct ib_device *, u8 port, struct ib_port_attr *props); +int qedr_query_port(struct ib_device *ibdev, u32 port, + struct ib_port_attr *props); -int qedr_iw_query_gid(struct ib_device *ibdev, u8 port, +int qedr_iw_query_gid(struct ib_device *ibdev, u32 port, int index, union ib_gid *gid); -int qedr_query_pkey(struct ib_device *, u8 port, u16 index, u16 *pkey); +int qedr_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey); int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata); void qedr_dealloc_ucontext(struct ib_ucontext *uctx); @@ -92,11 +93,11 @@ int qedr_post_send(struct ib_qp *, const struct ib_send_wr *, int qedr_post_recv(struct ib_qp *, const struct ib_recv_wr *, const struct ib_recv_wr **bad_wr); int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags, - u8 port_num, const struct ib_wc *in_wc, + u32 port_num, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const struct ib_mad *in_mad, struct ib_mad *out_mad, size_t *out_mad_size, u16 *out_mad_pkey_index); -int qedr_port_immutable(struct ib_device *ibdev, u8 port_num, +int qedr_port_immutable(struct ib_device *ibdev, u32 port_num, struct ib_port_immutable *immutable); #endif diff --git a/drivers/infiniband/hw/qib/qib.h b/drivers/infiniband/hw/qib/qib.h index ee211423058a..88497739029e 100644 --- a/drivers/infiniband/hw/qib/qib.h +++ b/drivers/infiniband/hw/qib/qib.h @@ -630,7 +630,7 @@ struct qib_pportdata { u8 rx_pol_inv; u8 hw_pidx; /* physical port index */ - u8 port; /* IB port number and index into dd->pports - 1 */ + u32 port; /* IB port number and index into dd->pports - 1 */ u8 delay_mult; @@ -1200,10 +1200,10 @@ static inline struct qib_pportdata *ppd_from_ibp(struct qib_ibport *ibp) return container_of(ibp, struct qib_pportdata, ibport_data); } -static inline struct qib_ibport *to_iport(struct ib_device *ibdev, u8 port) +static inline struct qib_ibport *to_iport(struct ib_device *ibdev, u32 port) { struct qib_devdata *dd = dd_from_ibdev(ibdev); - unsigned pidx = port - 1; /* IB number port from 1, hdw from 0 */ + u32 pidx = port - 1; /* IB number port from 1, hdw from 0 */ WARN_ON(pidx >= dd->num_pports); return &dd->pport[pidx].ibport_data; @@ -1303,11 +1303,6 @@ int qib_sdma_verbs_send(struct qib_pportdata *, struct rvt_sge_state *, /* ppd->sdma_lock should be locked before calling this. */ int qib_sdma_make_progress(struct qib_pportdata *dd); -static inline int qib_sdma_empty(const struct qib_pportdata *ppd) -{ - return ppd->sdma_descq_added == ppd->sdma_descq_removed; -} - /* must be called under qib_sdma_lock */ static inline u16 qib_sdma_descq_freecnt(const struct qib_pportdata *ppd) { @@ -1364,27 +1359,6 @@ static inline u32 qib_get_rcvhdrtail(const struct qib_ctxtdata *rcd) *((volatile __le64 *)rcd->rcvhdrtail_kvaddr)); /* DMA'ed */ } -static inline u32 qib_get_hdrqtail(const struct qib_ctxtdata *rcd) -{ - const struct qib_devdata *dd = rcd->dd; - u32 hdrqtail; - - if (dd->flags & QIB_NODMA_RTAIL) { - __le32 *rhf_addr; - u32 seq; - - rhf_addr = (__le32 *) rcd->rcvhdrq + - rcd->head + dd->rhf_offset; - seq = qib_hdrget_seq(rhf_addr); - hdrqtail = rcd->head; - if (seq == rcd->seq_cnt) - hdrqtail++; - } else - hdrqtail = qib_get_rcvhdrtail(rcd); - - return hdrqtail; -} - /* * sysfs interface. */ @@ -1395,7 +1369,7 @@ extern const struct attribute_group qib_attr_group; int qib_device_create(struct qib_devdata *); void qib_device_remove(struct qib_devdata *); -int qib_create_port_files(struct ib_device *ibdev, u8 port_num, +int qib_create_port_files(struct ib_device *ibdev, u32 port_num, struct kobject *kobj); void qib_verbs_unregister_sysfs(struct qib_devdata *); /* Hook for sysfs read of QSFP */ diff --git a/drivers/infiniband/hw/qib/qib_common.h b/drivers/infiniband/hw/qib/qib_common.h index f91f23e02283..cf652831d8e7 100644 --- a/drivers/infiniband/hw/qib/qib_common.h +++ b/drivers/infiniband/hw/qib/qib_common.h @@ -795,11 +795,4 @@ static inline __u32 qib_hdrget_use_egr_buf(const __le32 *rbuf) { return __le32_to_cpu(rbuf[0]) & QLOGIC_IB_RHF_L_USE_EGR; } - -static inline __u32 qib_hdrget_qib_ver(__le32 hdrword) -{ - return (__le32_to_cpu(hdrword) >> QLOGIC_IB_I_VERS_SHIFT) & - QLOGIC_IB_I_VERS_MASK; -} - #endif /* _QIB_COMMON_H */ diff --git a/drivers/infiniband/hw/qib/qib_file_ops.c b/drivers/infiniband/hw/qib/qib_file_ops.c index ff87a67dd7b7..c60e79d214a1 100644 --- a/drivers/infiniband/hw/qib/qib_file_ops.c +++ b/drivers/infiniband/hw/qib/qib_file_ops.c @@ -1758,7 +1758,8 @@ bail: } /** - * unlock_exptid - unlock any expected TID entries context still had in use + * unlock_expected_tids - unlock any expected TID entries context still had + * in use * @rcd: ctxt * * We don't actually update the chip here, because we do a bulk update @@ -2247,7 +2248,7 @@ static ssize_t qib_write_iter(struct kiocb *iocb, struct iov_iter *from) if (!iter_is_iovec(from) || !from->nr_segs || !pq) return -EINVAL; - + return qib_user_sdma_writev(rcd, pq, from->iov, from->nr_segs); } diff --git a/drivers/infiniband/hw/qib/qib_fs.c b/drivers/infiniband/hw/qib/qib_fs.c index e336d778e076..a0c5f3bdc324 100644 --- a/drivers/infiniband/hw/qib/qib_fs.c +++ b/drivers/infiniband/hw/qib/qib_fs.c @@ -427,79 +427,21 @@ bail: return ret; } -static int remove_file(struct dentry *parent, char *name) -{ - struct dentry *tmp; - int ret; - - tmp = lookup_one_len(name, parent, strlen(name)); - - if (IS_ERR(tmp)) { - ret = PTR_ERR(tmp); - goto bail; - } - - spin_lock(&tmp->d_lock); - if (simple_positive(tmp)) { - __d_drop(tmp); - spin_unlock(&tmp->d_lock); - simple_unlink(d_inode(parent), tmp); - } else { - spin_unlock(&tmp->d_lock); - } - dput(tmp); - - ret = 0; -bail: - /* - * We don't expect clients to care about the return value, but - * it's there if they need it. - */ - return ret; -} - static int remove_device_files(struct super_block *sb, struct qib_devdata *dd) { - struct dentry *dir, *root; + struct dentry *dir; char unit[10]; - int ret, i; - root = dget(sb->s_root); - inode_lock(d_inode(root)); snprintf(unit, sizeof(unit), "%u", dd->unit); - dir = lookup_one_len(unit, root, strlen(unit)); + dir = lookup_one_len_unlocked(unit, sb->s_root, strlen(unit)); if (IS_ERR(dir)) { - ret = PTR_ERR(dir); pr_err("Lookup of %s failed\n", unit); - goto bail; + return PTR_ERR(dir); } - - inode_lock(d_inode(dir)); - remove_file(dir, "counters"); - remove_file(dir, "counter_names"); - remove_file(dir, "portcounter_names"); - for (i = 0; i < dd->num_pports; i++) { - char fname[24]; - - sprintf(fname, "port%dcounters", i + 1); - remove_file(dir, fname); - if (dd->flags & QIB_HAS_QSFP) { - sprintf(fname, "qsfp%d", i + 1); - remove_file(dir, fname); - } - } - remove_file(dir, "flash"); - inode_unlock(d_inode(dir)); - ret = simple_rmdir(d_inode(root), dir); - d_drop(dir); - dput(dir); - -bail: - inode_unlock(d_inode(root)); - dput(root); - return ret; + simple_recursive_removal(dir, NULL); + return 0; } /* diff --git a/drivers/infiniband/hw/qib/qib_iba6120.c b/drivers/infiniband/hw/qib/qib_iba6120.c index b35e1174be22..a9b83bc13f4a 100644 --- a/drivers/infiniband/hw/qib/qib_iba6120.c +++ b/drivers/infiniband/hw/qib/qib_iba6120.c @@ -2609,7 +2609,7 @@ static void qib_chk_6120_errormask(struct qib_devdata *dd) } /** - * qib_get_faststats - get word counters from chip before they overflow + * qib_get_6120_faststats - get word counters from chip before they overflow * @t: contains a pointer to the qlogic_ib device qib_devdata * * This needs more work; in particular, decision on whether we really diff --git a/drivers/infiniband/hw/qib/qib_iba7220.c b/drivers/infiniband/hw/qib/qib_iba7220.c index 229dcd6ead95..d1c0bc31869f 100644 --- a/drivers/infiniband/hw/qib/qib_iba7220.c +++ b/drivers/infiniband/hw/qib/qib_iba7220.c @@ -2236,7 +2236,7 @@ static void qib_7220_tidtemplate(struct qib_devdata *dd) } /** - * qib_init_7220_get_base_info - set chip-specific flags for user code + * qib_7220_get_base_info - set chip-specific flags for user code * @rcd: the qlogic_ib ctxt * @kinfo: qib_base_info pointer * @@ -4411,7 +4411,7 @@ static void writescratch(struct qib_devdata *dd, u32 val) #define VALID_TS_RD_REG_MASK 0xBF /** - * qib_7220_tempsense_read - read register of temp sensor via TWSI + * qib_7220_tempsense_rd - read register of temp sensor via TWSI * @dd: the qlogic_ib device * @regnum: register to read from * diff --git a/drivers/infiniband/hw/qib/qib_iba7322.c b/drivers/infiniband/hw/qib/qib_iba7322.c index 9fe6ea75b45e..ab98b6a3ae1e 100644 --- a/drivers/infiniband/hw/qib/qib_iba7322.c +++ b/drivers/infiniband/hw/qib/qib_iba7322.c @@ -791,28 +791,6 @@ static inline u32 qib_read_ureg32(const struct qib_devdata *dd, } /** - * qib_read_ureg - read virtualized per-context register - * @dd: device - * @regno: register number - * @ctxt: context number - * - * Return the contents of a register that is virtualized to be per context. - * Returns -1 on errors (not distinguishable from valid contents at - * runtime; we may add a separate error variable at some point). - */ -static inline u64 qib_read_ureg(const struct qib_devdata *dd, - enum qib_ureg regno, int ctxt) -{ - - if (!dd->kregbase || !(dd->flags & QIB_PRESENT)) - return 0; - return readq(regno + (u64 __iomem *)( - (dd->ureg_align * ctxt) + (dd->userbase ? - (char __iomem *)dd->userbase : - (char __iomem *)dd->kregbase + dd->uregbase))); -} - -/** * qib_write_ureg - write virtualized per-context register * @dd: device * @regno: register number @@ -2513,7 +2491,7 @@ static int qib_7322_bringup_serdes(struct qib_pportdata *ppd) } /** - * qib_7322_quiet_serdes - set serdes to txidle + * qib_7322_mini_quiet_serdes - set serdes to txidle * @ppd: the qlogic_ib device * Called when driver is being unloaded */ @@ -3859,7 +3837,7 @@ static void qib_7322_tidtemplate(struct qib_devdata *dd) } /** - * qib_init_7322_get_base_info - set chip-specific flags for user code + * qib_7322_get_base_info - set chip-specific flags for user code * @rcd: the qlogic_ib ctxt * @kinfo: qib_base_info pointer * diff --git a/drivers/infiniband/hw/qib/qib_init.c b/drivers/infiniband/hw/qib/qib_init.c index 43c8ee1f46e0..b5a78576c48b 100644 --- a/drivers/infiniband/hw/qib/qib_init.c +++ b/drivers/infiniband/hw/qib/qib_init.c @@ -1609,7 +1609,7 @@ bail: } /** - * allocate eager buffers, both kernel and user contexts. + * qib_setup_eagerbufs - allocate eager buffers, both kernel and user contexts. * @rcd: the context we are setting up. * * Allocate the eager TID buffers and program them into hip. diff --git a/drivers/infiniband/hw/qib/qib_mad.c b/drivers/infiniband/hw/qib/qib_mad.c index 44e2f813024a..ef02f2bfddb2 100644 --- a/drivers/infiniband/hw/qib/qib_mad.c +++ b/drivers/infiniband/hw/qib/qib_mad.c @@ -203,7 +203,7 @@ static void qib_bad_mkey(struct qib_ibport *ibp, struct ib_smp *smp) /* * Send a Port Capability Mask Changed trap (ch. 14.3.11). */ -void qib_cap_mask_chg(struct rvt_dev_info *rdi, u8 port_num) +void qib_cap_mask_chg(struct rvt_dev_info *rdi, u32 port_num) { struct qib_ibdev *ibdev = container_of(rdi, struct qib_ibdev, rdi); struct qib_devdata *dd = dd_from_dev(ibdev); @@ -2360,7 +2360,7 @@ static int process_cc(struct ib_device *ibdev, int mad_flags, * * This is called by the ib_mad module. */ -int qib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port, +int qib_process_mad(struct ib_device *ibdev, int mad_flags, u32 port, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const struct ib_mad *in, struct ib_mad *out, size_t *out_mad_size, u16 *out_mad_pkey_index) diff --git a/drivers/infiniband/hw/qib/qib_qp.c b/drivers/infiniband/hw/qib/qib_qp.c index ca39a029e4af..1974ceb9d405 100644 --- a/drivers/infiniband/hw/qib/qib_qp.c +++ b/drivers/infiniband/hw/qib/qib_qp.c @@ -125,7 +125,7 @@ static void get_map_page(struct rvt_qpn_table *qpt, struct rvt_qpn_map *map) * zero/one for QP type IB_QPT_SMI/IB_QPT_GSI. */ int qib_alloc_qpn(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt, - enum ib_qp_type type, u8 port) + enum ib_qp_type type, u32 port) { u32 i, offset, max_scan, qpn; struct rvt_qpn_map *map; @@ -136,7 +136,7 @@ int qib_alloc_qpn(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt, u16 qpt_mask = dd->qpn_mask; if (type == IB_QPT_SMI || type == IB_QPT_GSI) { - unsigned n; + u32 n; ret = type == IB_QPT_GSI; n = 1 << (ret + 2 * (port - 1)); diff --git a/drivers/infiniband/hw/qib/qib_sd7220.c b/drivers/infiniband/hw/qib/qib_sd7220.c index 4f4a09c2dbcd..81b810d006c0 100644 --- a/drivers/infiniband/hw/qib/qib_sd7220.c +++ b/drivers/infiniband/hw/qib/qib_sd7220.c @@ -687,7 +687,6 @@ static int qib_sd7220_reg_mod(struct qib_devdata *dd, int sdnum, u32 loc, spin_unlock_irqrestore(&dd->cspec->sdepb_lock, flags); return -1; } - ret = 0; for (tries = EPB_TRANS_TRIES; tries; --tries) { transval = qib_read_kreg32(dd, trans); if (transval & EPB_TRANS_RDY) diff --git a/drivers/infiniband/hw/qib/qib_sysfs.c b/drivers/infiniband/hw/qib/qib_sysfs.c index 62c179fc764b..5e9e66f27064 100644 --- a/drivers/infiniband/hw/qib/qib_sysfs.c +++ b/drivers/infiniband/hw/qib/qib_sysfs.c @@ -728,7 +728,7 @@ const struct attribute_group qib_attr_group = { .attrs = qib_attributes, }; -int qib_create_port_files(struct ib_device *ibdev, u8 port_num, +int qib_create_port_files(struct ib_device *ibdev, u32 port_num, struct kobject *kobj) { struct qib_pportdata *ppd; diff --git a/drivers/infiniband/hw/qib/qib_verbs.c b/drivers/infiniband/hw/qib/qib_verbs.c index 8e0de265ad57..d17d034ecdfd 100644 --- a/drivers/infiniband/hw/qib/qib_verbs.c +++ b/drivers/infiniband/hw/qib/qib_verbs.c @@ -1188,7 +1188,7 @@ full: } } -static int qib_query_port(struct rvt_dev_info *rdi, u8 port_num, +static int qib_query_port(struct rvt_dev_info *rdi, u32 port_num, struct ib_port_attr *props) { struct qib_ibdev *ibdev = container_of(rdi, struct qib_ibdev, rdi); @@ -1273,7 +1273,7 @@ bail: return ret; } -static int qib_shut_down_port(struct rvt_dev_info *rdi, u8 port_num) +static int qib_shut_down_port(struct rvt_dev_info *rdi, u32 port_num) { struct qib_ibdev *ibdev = container_of(rdi, struct qib_ibdev, rdi); struct qib_devdata *dd = dd_from_dev(ibdev); @@ -1342,7 +1342,7 @@ struct ib_ah *qib_create_qp0_ah(struct qib_ibport *ibp, u16 dlid) struct rvt_qp *qp0; struct qib_pportdata *ppd = ppd_from_ibp(ibp); struct qib_devdata *dd = dd_from_ppd(ppd); - u8 port_num = ppd->port; + u32 port_num = ppd->port; memset(&attr, 0, sizeof(attr)); attr.type = rdma_ah_find_type(&dd->verbs_dev.rdi.ibdev, port_num); diff --git a/drivers/infiniband/hw/qib/qib_verbs.h b/drivers/infiniband/hw/qib/qib_verbs.h index dc0e81f3b6f4..07548fac1d8e 100644 --- a/drivers/infiniband/hw/qib/qib_verbs.h +++ b/drivers/infiniband/hw/qib/qib_verbs.h @@ -239,10 +239,10 @@ static inline int qib_pkey_ok(u16 pkey1, u16 pkey2) void qib_bad_pkey(struct qib_ibport *ibp, u32 key, u32 sl, u32 qp1, u32 qp2, __be16 lid1, __be16 lid2); -void qib_cap_mask_chg(struct rvt_dev_info *rdi, u8 port_num); +void qib_cap_mask_chg(struct rvt_dev_info *rdi, u32 port_num); void qib_sys_guid_chg(struct qib_ibport *ibp); void qib_node_desc_chg(struct qib_ibport *ibp); -int qib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, +int qib_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const struct ib_mad *in, struct ib_mad *out, size_t *out_mad_size, u16 *out_mad_pkey_index); @@ -273,7 +273,7 @@ void *qib_qp_priv_alloc(struct rvt_dev_info *rdi, struct rvt_qp *qp); void qib_qp_priv_free(struct rvt_dev_info *rdi, struct rvt_qp *qp); void qib_notify_qp_reset(struct rvt_qp *qp); int qib_alloc_qpn(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt, - enum ib_qp_type type, u8 port); + enum ib_qp_type type, u32 port); void qib_restart_rc(struct rvt_qp *qp, u32 psn, int wait); #ifdef CONFIG_DEBUG_FS diff --git a/drivers/infiniband/hw/usnic/usnic_ib_main.c b/drivers/infiniband/hw/usnic/usnic_ib_main.c index 1b63a491fa72..ff6a40e259d5 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_main.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_main.c @@ -303,7 +303,7 @@ static struct notifier_block usnic_ib_inetaddr_notifier = { }; /* End of inet section*/ -static int usnic_port_immutable(struct ib_device *ibdev, u8 port_num, +static int usnic_port_immutable(struct ib_device *ibdev, u32 port_num, struct ib_port_immutable *immutable) { struct ib_port_attr attr; diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c index 3705c6b8b223..57d210ca855a 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c @@ -270,7 +270,7 @@ static int create_qp_validate_user_data(struct usnic_ib_create_qp_cmd cmd) /* Start of ib callback functions */ enum rdma_link_layer usnic_ib_port_link_layer(struct ib_device *device, - u8 port_num) + u32 port_num) { return IB_LINK_LAYER_ETHERNET; } @@ -332,7 +332,7 @@ int usnic_ib_query_device(struct ib_device *ibdev, return 0; } -int usnic_ib_query_port(struct ib_device *ibdev, u8 port, +int usnic_ib_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props) { struct usnic_ib_dev *us_ibdev = to_usdev(ibdev); @@ -420,7 +420,7 @@ err_out: return err; } -int usnic_ib_query_gid(struct ib_device *ibdev, u8 port, int index, +int usnic_ib_query_gid(struct ib_device *ibdev, u32 port, int index, union ib_gid *gid) { diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.h b/drivers/infiniband/hw/usnic/usnic_ib_verbs.h index 11fe1ba6bbc9..6b82d0f2d184 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.h +++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.h @@ -37,16 +37,16 @@ #include "usnic_ib.h" enum rdma_link_layer usnic_ib_port_link_layer(struct ib_device *device, - u8 port_num); + u32 port_num); int usnic_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props, struct ib_udata *uhw); -int usnic_ib_query_port(struct ib_device *ibdev, u8 port, +int usnic_ib_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props); int usnic_ib_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr); -int usnic_ib_query_gid(struct ib_device *ibdev, u8 port, int index, +int usnic_ib_query_gid(struct ib_device *ibdev, u32 port, int index, union ib_gid *gid); int usnic_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); int usnic_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata); diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h index de57f2fed743..763ddc6f25d1 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h @@ -344,11 +344,6 @@ static inline enum ib_port_state pvrdma_port_state_to_ib( return (enum ib_port_state)state; } -static inline int ib_port_cap_flags_to_pvrdma(int flags) -{ - return flags & PVRDMA_MASK(PVRDMA_PORT_CAP_FLAGS_MAX); -} - static inline int pvrdma_port_cap_flags_to_ib(int flags) { return flags; @@ -410,11 +405,6 @@ static inline enum pvrdma_qp_type ib_qp_type_to_pvrdma(enum ib_qp_type type) return (enum pvrdma_qp_type)type; } -static inline enum ib_qp_type pvrdma_qp_type_to_ib(enum pvrdma_qp_type type) -{ - return (enum ib_qp_type)type; -} - static inline enum pvrdma_qp_state ib_qp_state_to_pvrdma(enum ib_qp_state state) { return (enum pvrdma_qp_state)state; diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c index 4b6019e7de67..6bf2d2e47d07 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c @@ -121,7 +121,7 @@ static int pvrdma_init_device(struct pvrdma_dev *dev) return 0; } -static int pvrdma_port_immutable(struct ib_device *ibdev, u8 port_num, +static int pvrdma_port_immutable(struct ib_device *ibdev, u32 port_num, struct ib_port_immutable *immutable) { struct pvrdma_dev *dev = to_vdev(ibdev); diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c index 1d3bdd7bb51d..67769b715126 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c @@ -882,7 +882,7 @@ out: } /** - * pvrdma_post_receive - post receive work request entries on a QP + * pvrdma_post_recv - post receive work request entries on a QP * @ibqp: the QP * @wr: the work request list to post * @bad_wr: the first bad WR returned diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c index fc412cbfd042..19176583dbde 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c @@ -125,7 +125,7 @@ int pvrdma_query_device(struct ib_device *ibdev, * * @return: 0 on success, otherwise negative errno */ -int pvrdma_query_port(struct ib_device *ibdev, u8 port, +int pvrdma_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props) { struct pvrdma_dev *dev = to_vdev(ibdev); @@ -183,7 +183,7 @@ int pvrdma_query_port(struct ib_device *ibdev, u8 port, * * @return: 0 on success, otherwise negative errno */ -int pvrdma_query_gid(struct ib_device *ibdev, u8 port, int index, +int pvrdma_query_gid(struct ib_device *ibdev, u32 port, int index, union ib_gid *gid) { struct pvrdma_dev *dev = to_vdev(ibdev); @@ -205,7 +205,7 @@ int pvrdma_query_gid(struct ib_device *ibdev, u8 port, int index, * * @return: 0 on success, otherwise negative errno */ -int pvrdma_query_pkey(struct ib_device *ibdev, u8 port, u16 index, +int pvrdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) { int err = 0; @@ -232,7 +232,7 @@ int pvrdma_query_pkey(struct ib_device *ibdev, u8 port, u16 index, } enum rdma_link_layer pvrdma_port_link_layer(struct ib_device *ibdev, - u8 port) + u32 port) { return IB_LINK_LAYER_ETHERNET; } @@ -274,7 +274,7 @@ int pvrdma_modify_device(struct ib_device *ibdev, int mask, * * @return: 0 on success, otherwise negative errno */ -int pvrdma_modify_port(struct ib_device *ibdev, u8 port, int mask, +int pvrdma_modify_port(struct ib_device *ibdev, u32 port, int mask, struct ib_port_modify *props) { struct ib_port_attr attr; @@ -516,7 +516,7 @@ int pvrdma_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, struct pvrdma_dev *dev = to_vdev(ibah->device); struct pvrdma_ah *ah = to_vah(ibah); const struct ib_global_route *grh; - u8 port_num = rdma_ah_get_port_num(ah_attr); + u32 port_num = rdma_ah_get_port_num(ah_attr); if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) return -EINVAL; diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h index 97ed8f952f6e..544b94d97c3a 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h @@ -70,30 +70,6 @@ enum pvrdma_mtu { PVRDMA_MTU_4096 = 5, }; -static inline int pvrdma_mtu_enum_to_int(enum pvrdma_mtu mtu) -{ - switch (mtu) { - case PVRDMA_MTU_256: return 256; - case PVRDMA_MTU_512: return 512; - case PVRDMA_MTU_1024: return 1024; - case PVRDMA_MTU_2048: return 2048; - case PVRDMA_MTU_4096: return 4096; - default: return -1; - } -} - -static inline enum pvrdma_mtu pvrdma_mtu_int_to_enum(int mtu) -{ - switch (mtu) { - case 256: return PVRDMA_MTU_256; - case 512: return PVRDMA_MTU_512; - case 1024: return PVRDMA_MTU_1024; - case 2048: return PVRDMA_MTU_2048; - case 4096: - default: return PVRDMA_MTU_4096; - } -} - enum pvrdma_port_state { PVRDMA_PORT_NOP = 0, PVRDMA_PORT_DOWN = 1, @@ -138,17 +114,6 @@ enum pvrdma_port_width { PVRDMA_WIDTH_12X = 8, }; -static inline int pvrdma_width_enum_to_int(enum pvrdma_port_width width) -{ - switch (width) { - case PVRDMA_WIDTH_1X: return 1; - case PVRDMA_WIDTH_4X: return 4; - case PVRDMA_WIDTH_8X: return 8; - case PVRDMA_WIDTH_12X: return 12; - default: return -1; - } -} - enum pvrdma_port_speed { PVRDMA_SPEED_SDR = 1, PVRDMA_SPEED_DDR = 2, @@ -383,17 +348,17 @@ enum pvrdma_access_flags { int pvrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *props, struct ib_udata *udata); -int pvrdma_query_port(struct ib_device *ibdev, u8 port, +int pvrdma_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props); -int pvrdma_query_gid(struct ib_device *ibdev, u8 port, +int pvrdma_query_gid(struct ib_device *ibdev, u32 port, int index, union ib_gid *gid); -int pvrdma_query_pkey(struct ib_device *ibdev, u8 port, +int pvrdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey); enum rdma_link_layer pvrdma_port_link_layer(struct ib_device *ibdev, - u8 port); + u32 port); int pvrdma_modify_device(struct ib_device *ibdev, int mask, struct ib_device_modify *props); -int pvrdma_modify_port(struct ib_device *ibdev, u8 port, +int pvrdma_modify_port(struct ib_device *ibdev, u32 port, int mask, struct ib_port_modify *props); int pvrdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma); int pvrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata); |