diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2023-09-02 02:49:33 +0300 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2023-09-02 02:49:33 +0300 |
commit | f7e97ce26972ae7be8bbbae8d819ff311d4c5900 (patch) | |
tree | 6750e2bab2c7b3fafc30d9bd2cbe2be7645c7ac5 /drivers/infiniband/hw/bnxt_re | |
parent | 2fcbb03847d89155d7b33d75ffee3a6bc5c51c97 (diff) | |
parent | f5acc36b0714b7b8510a8b436087d33a65cb05f4 (diff) | |
download | linux-f7e97ce26972ae7be8bbbae8d819ff311d4c5900.tar.xz |
Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma
Pull rdma updates from Jason Gunthorpe:
"Many small changes across the subystem, some highlights:
- Usual driver cleanups in qedr, siw, erdma, hfi1, mlx4/5, irdma,
mthca, hns, and bnxt_re
- siw now works over tunnel and other netdevs with a MAC address by
removing assumptions about a MAC/GID from the connection manager
- "Doorbell Pacing" for bnxt_re - this is a best effort scheme to
allow userspace to slow down the doorbell rings if the HW gets full
- irdma egress VLAN priority, better QP/WQ sizing
- rxe bug fixes in queue draining and srq resizing
- Support more ethernet speed options in the core layer
- DMABUF support for bnxt_re
- Multi-stage MTT support for erdma to allow much bigger MR
registrations
- A irdma fix with a CVE that came in too late to go to -rc, missing
bounds checking for 0 length MRs"
* tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma: (87 commits)
IB/hfi1: Reduce printing of errors during driver shut down
RDMA/hfi1: Move user SDMA system memory pinning code to its own file
RDMA/hfi1: Use list_for_each_entry() helper
RDMA/mlx5: Fix trailing */ formatting in block comment
RDMA/rxe: Fix redundant break statement in switch-case.
RDMA/efa: Fix wrong resources deallocation order
RDMA/siw: Call llist_reverse_order in siw_run_sq
RDMA/siw: Correct wrong debug message
RDMA/siw: Balance the reference of cep->kref in the error path
Revert "IB/isert: Fix incorrect release of isert connection"
RDMA/bnxt_re: Fix kernel doc errors
RDMA/irdma: Prevent zero-length STAG registration
RDMA/erdma: Implement hierarchical MTT
RDMA/erdma: Refactor the storage structure of MTT entries
RDMA/erdma: Renaming variable names and field names of struct erdma_mem
RDMA/hns: Support hns HW stats
RDMA/hns: Dump whole QP/CQ/MR resource in raw
RDMA/irdma: Add missing kernel-doc in irdma_setup_umode_qp()
RDMA/mlx4: Copy union directly
RDMA/irdma: Drop unused kernel push code
...
Diffstat (limited to 'drivers/infiniband/hw/bnxt_re')
-rw-r--r-- | drivers/infiniband/hw/bnxt_re/bnxt_re.h | 35 | ||||
-rw-r--r-- | drivers/infiniband/hw/bnxt_re/hw_counters.c | 84 | ||||
-rw-r--r-- | drivers/infiniband/hw/bnxt_re/hw_counters.h | 55 | ||||
-rw-r--r-- | drivers/infiniband/hw/bnxt_re/ib_verbs.c | 255 | ||||
-rw-r--r-- | drivers/infiniband/hw/bnxt_re/ib_verbs.h | 6 | ||||
-rw-r--r-- | drivers/infiniband/hw/bnxt_re/main.c | 277 | ||||
-rw-r--r-- | drivers/infiniband/hw/bnxt_re/qplib_fp.c | 47 | ||||
-rw-r--r-- | drivers/infiniband/hw/bnxt_re/qplib_rcfw.c | 66 | ||||
-rw-r--r-- | drivers/infiniband/hw/bnxt_re/qplib_res.c | 38 | ||||
-rw-r--r-- | drivers/infiniband/hw/bnxt_re/qplib_res.h | 23 | ||||
-rw-r--r-- | drivers/infiniband/hw/bnxt_re/qplib_sp.c | 85 | ||||
-rw-r--r-- | drivers/infiniband/hw/bnxt_re/qplib_sp.h | 2 |
12 files changed, 728 insertions, 245 deletions
diff --git a/drivers/infiniband/hw/bnxt_re/bnxt_re.h b/drivers/infiniband/hw/bnxt_re/bnxt_re.h index ea81b2497511..9fd9849ebdd1 100644 --- a/drivers/infiniband/hw/bnxt_re/bnxt_re.h +++ b/drivers/infiniband/hw/bnxt_re/bnxt_re.h @@ -112,12 +112,34 @@ struct bnxt_re_gsi_context { #define BNXT_RE_NQ_IDX 1 #define BNXT_RE_GEN_P5_MAX_VF 64 +struct bnxt_re_pacing { + u64 dbr_db_fifo_reg_off; + void *dbr_page; + u64 dbr_bar_addr; + u32 pacing_algo_th; + u32 do_pacing_save; + u32 dbq_pacing_time; /* ms */ + u32 dbr_def_do_pacing; + bool dbr_pacing; + struct mutex dbq_lock; /* synchronize db pacing algo */ +}; + +#define BNXT_RE_MAX_DBR_DO_PACING 0xFFFF +#define BNXT_RE_DBR_PACING_TIME 5 /* ms */ +#define BNXT_RE_PACING_ALGO_THRESHOLD 250 /* Entries in DB FIFO */ +#define BNXT_RE_PACING_ALARM_TH_MULTIPLE 2 /* Multiple of pacing algo threshold */ +/* Default do_pacing value when there is no congestion */ +#define BNXT_RE_DBR_DO_PACING_NO_CONGESTION 0x7F /* 1 in 512 probability */ +#define BNXT_RE_DB_FIFO_ROOM_MASK 0x1FFF8000 +#define BNXT_RE_MAX_FIFO_DEPTH 0x2c00 +#define BNXT_RE_DB_FIFO_ROOM_SHIFT 15 +#define BNXT_RE_GRC_FIFO_REG_BASE 0x2000 + struct bnxt_re_dev { struct ib_device ibdev; struct list_head list; unsigned long flags; #define BNXT_RE_FLAG_NETDEV_REGISTERED 0 -#define BNXT_RE_FLAG_GOT_MSIX 2 #define BNXT_RE_FLAG_HAVE_L2_REF 3 #define BNXT_RE_FLAG_RCFW_CHANNEL_EN 4 #define BNXT_RE_FLAG_QOS_WORK_REG 5 @@ -152,16 +174,9 @@ struct bnxt_re_dev { struct bnxt_qplib_res qplib_res; struct bnxt_qplib_dpi dpi_privileged; - atomic_t qp_count; struct mutex qp_lock; /* protect qp list */ struct list_head qp_list; - atomic_t cq_count; - atomic_t srq_count; - atomic_t mr_count; - atomic_t mw_count; - atomic_t ah_count; - atomic_t pd_count; /* Max of 2 lossless traffic class supported per port */ u16 cosq[2]; @@ -171,6 +186,9 @@ struct bnxt_re_dev { atomic_t nq_alloc_cnt; u32 is_virtfn; u32 num_vfs; + struct bnxt_re_pacing pacing; + struct work_struct dbq_fifo_check_work; + struct delayed_work dbq_pacing_work; }; #define to_bnxt_re_dev(ptr, member) \ @@ -181,6 +199,7 @@ struct bnxt_re_dev { #define BNXT_RE_ROCEV2_IPV6_PACKET 3 #define BNXT_RE_CHECK_RC(x) ((x) && ((x) != -ETIMEDOUT)) +void bnxt_re_pacing_alert(struct bnxt_re_dev *rdev); static inline struct device *rdev_to_dev(struct bnxt_re_dev *rdev) { diff --git a/drivers/infiniband/hw/bnxt_re/hw_counters.c b/drivers/infiniband/hw/bnxt_re/hw_counters.c index 825d512799d9..93572405d6fa 100644 --- a/drivers/infiniband/hw/bnxt_re/hw_counters.c +++ b/drivers/infiniband/hw/bnxt_re/hw_counters.c @@ -61,15 +61,29 @@ static const struct rdma_stat_desc bnxt_re_stat_descs[] = { [BNXT_RE_ACTIVE_PD].name = "active_pds", [BNXT_RE_ACTIVE_AH].name = "active_ahs", [BNXT_RE_ACTIVE_QP].name = "active_qps", + [BNXT_RE_ACTIVE_RC_QP].name = "active_rc_qps", + [BNXT_RE_ACTIVE_UD_QP].name = "active_ud_qps", [BNXT_RE_ACTIVE_SRQ].name = "active_srqs", [BNXT_RE_ACTIVE_CQ].name = "active_cqs", [BNXT_RE_ACTIVE_MR].name = "active_mrs", [BNXT_RE_ACTIVE_MW].name = "active_mws", + [BNXT_RE_WATERMARK_PD].name = "watermark_pds", + [BNXT_RE_WATERMARK_AH].name = "watermark_ahs", + [BNXT_RE_WATERMARK_QP].name = "watermark_qps", + [BNXT_RE_WATERMARK_RC_QP].name = "watermark_rc_qps", + [BNXT_RE_WATERMARK_UD_QP].name = "watermark_ud_qps", + [BNXT_RE_WATERMARK_SRQ].name = "watermark_srqs", + [BNXT_RE_WATERMARK_CQ].name = "watermark_cqs", + [BNXT_RE_WATERMARK_MR].name = "watermark_mrs", + [BNXT_RE_WATERMARK_MW].name = "watermark_mws", + [BNXT_RE_RESIZE_CQ_CNT].name = "resize_cq_cnt", [BNXT_RE_RX_PKTS].name = "rx_pkts", [BNXT_RE_RX_BYTES].name = "rx_bytes", [BNXT_RE_TX_PKTS].name = "tx_pkts", [BNXT_RE_TX_BYTES].name = "tx_bytes", [BNXT_RE_RECOVERABLE_ERRORS].name = "recoverable_errors", + [BNXT_RE_TX_ERRORS].name = "tx_roce_errors", + [BNXT_RE_TX_DISCARDS].name = "tx_roce_discards", [BNXT_RE_RX_ERRORS].name = "rx_roce_errors", [BNXT_RE_RX_DISCARDS].name = "rx_roce_discards", [BNXT_RE_TO_RETRANSMITS].name = "to_retransmits", @@ -117,14 +131,25 @@ static const struct rdma_stat_desc bnxt_re_stat_descs[] = { [BNXT_RE_TX_READ_RES].name = "tx_read_resp", [BNXT_RE_TX_WRITE_REQ].name = "tx_write_req", [BNXT_RE_TX_SEND_REQ].name = "tx_send_req", + [BNXT_RE_TX_ROCE_PKTS].name = "tx_roce_only_pkts", + [BNXT_RE_TX_ROCE_BYTES].name = "tx_roce_only_bytes", [BNXT_RE_RX_ATOMIC_REQ].name = "rx_atomic_req", [BNXT_RE_RX_READ_REQ].name = "rx_read_req", [BNXT_RE_RX_READ_RESP].name = "rx_read_resp", [BNXT_RE_RX_WRITE_REQ].name = "rx_write_req", [BNXT_RE_RX_SEND_REQ].name = "rx_send_req", + [BNXT_RE_RX_ROCE_PKTS].name = "rx_roce_only_pkts", + [BNXT_RE_RX_ROCE_BYTES].name = "rx_roce_only_bytes", [BNXT_RE_RX_ROCE_GOOD_PKTS].name = "rx_roce_good_pkts", [BNXT_RE_RX_ROCE_GOOD_BYTES].name = "rx_roce_good_bytes", - [BNXT_RE_OOB].name = "rx_out_of_buffer" + [BNXT_RE_OOB].name = "rx_out_of_buffer", + [BNXT_RE_TX_CNP].name = "tx_cnp_pkts", + [BNXT_RE_RX_CNP].name = "rx_cnp_pkts", + [BNXT_RE_RX_ECN].name = "rx_ecn_marked_pkts", + [BNXT_RE_PACING_RESCHED].name = "pacing_reschedule", + [BNXT_RE_PACING_CMPL].name = "pacing_complete", + [BNXT_RE_PACING_ALERT].name = "pacing_alerts", + [BNXT_RE_DB_FIFO_REG].name = "db_fifo_register", }; static void bnxt_re_copy_ext_stats(struct bnxt_re_dev *rdev, @@ -136,14 +161,22 @@ static void bnxt_re_copy_ext_stats(struct bnxt_re_dev *rdev, stats->value[BNXT_RE_TX_READ_RES] = s->tx_read_res; stats->value[BNXT_RE_TX_WRITE_REQ] = s->tx_write_req; stats->value[BNXT_RE_TX_SEND_REQ] = s->tx_send_req; + stats->value[BNXT_RE_TX_ROCE_PKTS] = s->tx_roce_pkts; + stats->value[BNXT_RE_TX_ROCE_BYTES] = s->tx_roce_bytes; stats->value[BNXT_RE_RX_ATOMIC_REQ] = s->rx_atomic_req; stats->value[BNXT_RE_RX_READ_REQ] = s->rx_read_req; stats->value[BNXT_RE_RX_READ_RESP] = s->rx_read_res; stats->value[BNXT_RE_RX_WRITE_REQ] = s->rx_write_req; stats->value[BNXT_RE_RX_SEND_REQ] = s->rx_send_req; + stats->value[BNXT_RE_RX_ROCE_PKTS] = s->rx_roce_pkts; + stats->value[BNXT_RE_RX_ROCE_BYTES] = s->rx_roce_bytes; stats->value[BNXT_RE_RX_ROCE_GOOD_PKTS] = s->rx_roce_good_pkts; stats->value[BNXT_RE_RX_ROCE_GOOD_BYTES] = s->rx_roce_good_bytes; stats->value[BNXT_RE_OOB] = s->rx_out_of_buffer; + stats->value[BNXT_RE_TX_CNP] = s->tx_cnp; + stats->value[BNXT_RE_RX_CNP] = s->rx_cnp; + stats->value[BNXT_RE_RX_ECN] = s->rx_ecn_marked; + stats->value[BNXT_RE_OUT_OF_SEQ_ERR] = s->rx_out_of_sequence; } static int bnxt_re_get_ext_stat(struct bnxt_re_dev *rdev, @@ -249,30 +282,59 @@ static void bnxt_re_copy_err_stats(struct bnxt_re_dev *rdev, err_s->res_oos_drop_count; } +static void bnxt_re_copy_db_pacing_stats(struct bnxt_re_dev *rdev, + struct rdma_hw_stats *stats) +{ + struct bnxt_re_db_pacing_stats *pacing_s = &rdev->stats.pacing; + + stats->value[BNXT_RE_PACING_RESCHED] = pacing_s->resched; + stats->value[BNXT_RE_PACING_CMPL] = pacing_s->complete; + stats->value[BNXT_RE_PACING_ALERT] = pacing_s->alerts; + stats->value[BNXT_RE_DB_FIFO_REG] = + readl(rdev->en_dev->bar0 + rdev->pacing.dbr_db_fifo_reg_off); +} + int bnxt_re_ib_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, u32 port, int index) { struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); - struct ctx_hw_stats *hw_stats = NULL; + struct bnxt_re_res_cntrs *res_s = &rdev->stats.res; struct bnxt_qplib_roce_stats *err_s = NULL; + struct ctx_hw_stats *hw_stats = NULL; int rc = 0; hw_stats = rdev->qplib_ctx.stats.dma; if (!port || !stats) return -EINVAL; - stats->value[BNXT_RE_ACTIVE_QP] = atomic_read(&rdev->qp_count); - stats->value[BNXT_RE_ACTIVE_SRQ] = atomic_read(&rdev->srq_count); - stats->value[BNXT_RE_ACTIVE_CQ] = atomic_read(&rdev->cq_count); - stats->value[BNXT_RE_ACTIVE_MR] = atomic_read(&rdev->mr_count); - stats->value[BNXT_RE_ACTIVE_MW] = atomic_read(&rdev->mw_count); - stats->value[BNXT_RE_ACTIVE_PD] = atomic_read(&rdev->pd_count); - stats->value[BNXT_RE_ACTIVE_AH] = atomic_read(&rdev->ah_count); + stats->value[BNXT_RE_ACTIVE_QP] = atomic_read(&res_s->qp_count); + stats->value[BNXT_RE_ACTIVE_RC_QP] = atomic_read(&res_s->rc_qp_count); + stats->value[BNXT_RE_ACTIVE_UD_QP] = atomic_read(&res_s->ud_qp_count); + stats->value[BNXT_RE_ACTIVE_SRQ] = atomic_read(&res_s->srq_count); + stats->value[BNXT_RE_ACTIVE_CQ] = atomic_read(&res_s->cq_count); + stats->value[BNXT_RE_ACTIVE_MR] = atomic_read(&res_s->mr_count); + stats->value[BNXT_RE_ACTIVE_MW] = atomic_read(&res_s->mw_count); + stats->value[BNXT_RE_ACTIVE_PD] = atomic_read(&res_s->pd_count); + stats->value[BNXT_RE_ACTIVE_AH] = atomic_read(&res_s->ah_count); + stats->value[BNXT_RE_WATERMARK_QP] = res_s->qp_watermark; + stats->value[BNXT_RE_WATERMARK_RC_QP] = res_s->rc_qp_watermark; + stats->value[BNXT_RE_WATERMARK_UD_QP] = res_s->ud_qp_watermark; + stats->value[BNXT_RE_WATERMARK_SRQ] = res_s->srq_watermark; + stats->value[BNXT_RE_WATERMARK_CQ] = res_s->cq_watermark; + stats->value[BNXT_RE_WATERMARK_MR] = res_s->mr_watermark; + stats->value[BNXT_RE_WATERMARK_MW] = res_s->mw_watermark; + stats->value[BNXT_RE_WATERMARK_PD] = res_s->pd_watermark; + stats->value[BNXT_RE_WATERMARK_AH] = res_s->ah_watermark; + stats->value[BNXT_RE_RESIZE_CQ_CNT] = atomic_read(&res_s->resize_count); if (hw_stats) { stats->value[BNXT_RE_RECOVERABLE_ERRORS] = le64_to_cpu(hw_stats->tx_bcast_pkts); + stats->value[BNXT_RE_TX_DISCARDS] = + le64_to_cpu(hw_stats->tx_discard_pkts); + stats->value[BNXT_RE_TX_ERRORS] = + le64_to_cpu(hw_stats->tx_error_pkts); stats->value[BNXT_RE_RX_ERRORS] = le64_to_cpu(hw_stats->rx_error_pkts); stats->value[BNXT_RE_RX_DISCARDS] = @@ -294,6 +356,7 @@ int bnxt_re_ib_get_hw_stats(struct ib_device *ibdev, &rdev->flags); goto done; } + bnxt_re_copy_err_stats(rdev, stats, err_s); if (_is_ext_stats_supported(rdev->dev_attr.dev_cap_flags) && !rdev->is_virtfn) { rc = bnxt_re_get_ext_stat(rdev, stats); @@ -303,7 +366,8 @@ int bnxt_re_ib_get_hw_stats(struct ib_device *ibdev, goto done; } } - bnxt_re_copy_err_stats(rdev, stats, err_s); + if (rdev->pacing.dbr_pacing) + bnxt_re_copy_db_pacing_stats(rdev, stats); } done: diff --git a/drivers/infiniband/hw/bnxt_re/hw_counters.h b/drivers/infiniband/hw/bnxt_re/hw_counters.h index 7943b2c393e4..e541b6f8ca9f 100644 --- a/drivers/infiniband/hw/bnxt_re/hw_counters.h +++ b/drivers/infiniband/hw/bnxt_re/hw_counters.h @@ -44,15 +44,29 @@ enum bnxt_re_hw_stats { BNXT_RE_ACTIVE_PD, BNXT_RE_ACTIVE_AH, BNXT_RE_ACTIVE_QP, + BNXT_RE_ACTIVE_RC_QP, + BNXT_RE_ACTIVE_UD_QP, BNXT_RE_ACTIVE_SRQ, BNXT_RE_ACTIVE_CQ, BNXT_RE_ACTIVE_MR, BNXT_RE_ACTIVE_MW, + BNXT_RE_WATERMARK_PD, + BNXT_RE_WATERMARK_AH, + BNXT_RE_WATERMARK_QP, + BNXT_RE_WATERMARK_RC_QP, + BNXT_RE_WATERMARK_UD_QP, + BNXT_RE_WATERMARK_SRQ, + BNXT_RE_WATERMARK_CQ, + BNXT_RE_WATERMARK_MR, + BNXT_RE_WATERMARK_MW, + BNXT_RE_RESIZE_CQ_CNT, BNXT_RE_RX_PKTS, BNXT_RE_RX_BYTES, BNXT_RE_TX_PKTS, BNXT_RE_TX_BYTES, BNXT_RE_RECOVERABLE_ERRORS, + BNXT_RE_TX_ERRORS, + BNXT_RE_TX_DISCARDS, BNXT_RE_RX_ERRORS, BNXT_RE_RX_DISCARDS, BNXT_RE_TO_RETRANSMITS, @@ -100,19 +114,58 @@ enum bnxt_re_hw_stats { BNXT_RE_TX_READ_RES, BNXT_RE_TX_WRITE_REQ, BNXT_RE_TX_SEND_REQ, + BNXT_RE_TX_ROCE_PKTS, + BNXT_RE_TX_ROCE_BYTES, BNXT_RE_RX_ATOMIC_REQ, BNXT_RE_RX_READ_REQ, BNXT_RE_RX_READ_RESP, BNXT_RE_RX_WRITE_REQ, BNXT_RE_RX_SEND_REQ, + BNXT_RE_RX_ROCE_PKTS, + BNXT_RE_RX_ROCE_BYTES, BNXT_RE_RX_ROCE_GOOD_PKTS, BNXT_RE_RX_ROCE_GOOD_BYTES, BNXT_RE_OOB, + BNXT_RE_TX_CNP, + BNXT_RE_RX_CNP, + BNXT_RE_RX_ECN, + BNXT_RE_PACING_RESCHED, + BNXT_RE_PACING_CMPL, + BNXT_RE_PACING_ALERT, + BNXT_RE_DB_FIFO_REG, BNXT_RE_NUM_EXT_COUNTERS }; #define BNXT_RE_NUM_STD_COUNTERS (BNXT_RE_OUT_OF_SEQ_ERR + 1) +struct bnxt_re_db_pacing_stats { + u64 resched; + u64 complete; + u64 alerts; +}; + +struct bnxt_re_res_cntrs { + atomic_t qp_count; + atomic_t rc_qp_count; + atomic_t ud_qp_count; + atomic_t cq_count; + atomic_t srq_count; + atomic_t mr_count; + atomic_t mw_count; + atomic_t ah_count; + atomic_t pd_count; + atomic_t resize_count; + u64 qp_watermark; + u64 rc_qp_watermark; + u64 ud_qp_watermark; + u64 cq_watermark; + u64 srq_watermark; + u64 mr_watermark; + u64 mw_watermark; + u64 ah_watermark; + u64 pd_watermark; +}; + struct bnxt_re_rstat { struct bnxt_qplib_roce_stats errs; struct bnxt_qplib_ext_stat ext_stat; @@ -120,6 +173,8 @@ struct bnxt_re_rstat { struct bnxt_re_stats { struct bnxt_re_rstat rstat; + struct bnxt_re_res_cntrs res; + struct bnxt_re_db_pacing_stats pacing; }; struct rdma_hw_stats *bnxt_re_ib_alloc_hw_port_stats(struct ib_device *ibdev, diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 03cc45a5458d..0848c2c2ffcf 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -284,7 +284,7 @@ int bnxt_re_query_gid(struct ib_device *ibdev, u32 port_num, int index, union ib_gid *gid) { struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); - int rc = 0; + int rc; /* Ignore port_num */ memset(gid, 0, sizeof(*gid)); @@ -565,6 +565,8 @@ bnxt_re_mmap_entry_insert(struct bnxt_re_ucontext *uctx, u64 mem_offset, break; case BNXT_RE_MMAP_UC_DB: case BNXT_RE_MMAP_WC_DB: + case BNXT_RE_MMAP_DBR_BAR: + case BNXT_RE_MMAP_DBR_PAGE: ret = rdma_user_mmap_entry_insert(&uctx->ib_uctx, &entry->rdma_entry, PAGE_SIZE); break; @@ -600,7 +602,7 @@ int bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata) if (!bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl, &pd->qplib_pd)) - atomic_dec(&rdev->pd_count); + atomic_dec(&rdev->stats.res.pd_count); } return 0; } @@ -613,10 +615,11 @@ int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) udata, struct bnxt_re_ucontext, ib_uctx); struct bnxt_re_pd *pd = container_of(ibpd, struct bnxt_re_pd, ib_pd); struct bnxt_re_user_mmap_entry *entry = NULL; + u32 active_pds; int rc = 0; pd->rdev = rdev; - if (bnxt_qplib_alloc_pd(&rdev->qplib_res.pd_tbl, &pd->qplib_pd)) { + if (bnxt_qplib_alloc_pd(&rdev->qplib_res, &pd->qplib_pd)) { ibdev_err(&rdev->ibdev, "Failed to allocate HW PD"); rc = -ENOMEM; goto fail; @@ -663,7 +666,9 @@ int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) if (bnxt_re_create_fence_mr(pd)) ibdev_warn(&rdev->ibdev, "Failed to create Fence-MR\n"); - atomic_inc(&rdev->pd_count); + active_pds = atomic_inc_return(&rdev->stats.res.pd_count); + if (active_pds > rdev->stats.res.pd_watermark) + rdev->stats.res.pd_watermark = active_pds; return 0; dbfail: @@ -679,7 +684,7 @@ int bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags) struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah); struct bnxt_re_dev *rdev = ah->rdev; bool block = true; - int rc = 0; + int rc; block = !(flags & RDMA_DESTROY_AH_SLEEPABLE); rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah, block); @@ -689,7 +694,7 @@ int bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags) else goto fail; } - atomic_dec(&rdev->ah_count); + atomic_dec(&rdev->stats.res.ah_count); fail: return rc; } @@ -723,6 +728,7 @@ int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr, const struct ib_gid_attr *sgid_attr; struct bnxt_re_gid_ctx *ctx; struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah); + u32 active_ahs; u8 nw_type; int rc; @@ -775,7 +781,9 @@ int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr, wmb(); /* make sure cache is updated. */ spin_unlock_irqrestore(&uctx->sh_lock, flag); } - atomic_inc(&rdev->ah_count); + active_ahs = atomic_inc_return(&rdev->stats.res.ah_count); + if (active_ahs > rdev->stats.res.ah_watermark) + rdev->stats.res.ah_watermark = active_ahs; return 0; } @@ -826,7 +834,7 @@ static int bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp *qp) struct bnxt_re_qp *gsi_sqp; struct bnxt_re_ah *gsi_sah; struct bnxt_re_dev *rdev; - int rc = 0; + int rc; rdev = qp->rdev; gsi_sqp = rdev->gsi_ctx.gsi_sqp; @@ -836,7 +844,7 @@ static int bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp *qp) bnxt_qplib_destroy_ah(&rdev->qplib_res, &gsi_sah->qplib_ah, true); - atomic_dec(&rdev->ah_count); + atomic_dec(&rdev->stats.res.ah_count); bnxt_qplib_clean_qp(&qp->qplib_qp); ibdev_dbg(&rdev->ibdev, "Destroy the shadow QP\n"); @@ -851,7 +859,7 @@ static int bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp *qp) mutex_lock(&rdev->qp_lock); list_del(&gsi_sqp->list); mutex_unlock(&rdev->qp_lock); - atomic_dec(&rdev->qp_count); + atomic_dec(&rdev->stats.res.qp_count); kfree(rdev->gsi_ctx.sqp_tbl); kfree(gsi_sah); @@ -901,7 +909,7 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) mutex_lock(&rdev->qp_lock); list_del(&qp->list); mutex_unlock(&rdev->qp_lock); - atomic_dec(&rdev->qp_count); + atomic_dec(&rdev->stats.res.qp_count); ib_umem_release(qp->rumem); ib_umem_release(qp->sumem); @@ -1095,7 +1103,7 @@ static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah "Failed to allocate HW AH for Shadow QP"); goto fail; } - atomic_inc(&rdev->ah_count); + atomic_inc(&rdev->stats.res.ah_count); return ah; @@ -1163,7 +1171,7 @@ static struct bnxt_re_qp *bnxt_re_create_shadow_qp INIT_LIST_HEAD(&qp->list); mutex_lock(&rdev->qp_lock); list_add_tail(&qp->list, &rdev->qp_list); - atomic_inc(&rdev->qp_count); + atomic_inc(&rdev->stats.res.qp_count); mutex_unlock(&rdev->qp_lock); return qp; fail: @@ -1340,8 +1348,7 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, qplqp->pd = &pd->qplib_pd; qplqp->qp_handle = (u64)qplqp; qplqp->max_inline_data = init_attr->cap.max_inline_data; - qplqp->sig_type = ((init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? - true : false); + qplqp->sig_type = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR; qptype = bnxt_re_init_qp_type(rdev, init_attr); if (qptype < 0) { rc = qptype; @@ -1446,7 +1453,7 @@ static int bnxt_re_create_gsi_qp(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, { struct bnxt_re_dev *rdev; struct bnxt_qplib_qp *qplqp; - int rc = 0; + int rc; rdev = qp->rdev; qplqp = &qp->qplib_qp; @@ -1497,6 +1504,7 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, struct bnxt_re_dev *rdev = pd->rdev; struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); + u32 active_qps; int rc; rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr); @@ -1545,7 +1553,18 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, mutex_lock(&rdev->qp_lock); list_add_tail(&qp->list, &rdev->qp_list); mutex_unlock(&rdev->qp_lock); - atomic_inc(&rdev->qp_count); + active_qps = atomic_inc_return(&rdev->stats.res.qp_count); + if (active_qps > rdev->stats.res.qp_watermark) + rdev->stats.res.qp_watermark = active_qps; + if (qp_init_attr->qp_type == IB_QPT_RC) { + active_qps = atomic_inc_return(&rdev->stats.res.rc_qp_count); + if (active_qps > rdev->stats.res.rc_qp_watermark) + rdev->stats.res.rc_qp_watermark = active_qps; + } else if (qp_init_attr->qp_type == IB_QPT_UD) { + active_qps = atomic_inc_return(&rdev->stats.res.ud_qp_count); + if (active_qps > rdev->stats.res.ud_qp_watermark) + rdev->stats.res.ud_qp_watermark = active_qps; + } return 0; qp_destroy: @@ -1648,7 +1667,7 @@ int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata) nq = qplib_srq->cq->nq; bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq); ib_umem_release(srq->umem); - atomic_dec(&rdev->srq_count); + atomic_dec(&rdev->stats.res.srq_count); if (nq) nq->budget--; return 0; @@ -1696,6 +1715,7 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq, struct bnxt_re_srq *srq; struct bnxt_re_pd *pd; struct ib_pd *ib_pd; + u32 active_srqs; int rc, entries; ib_pd = ib_srq->pd; @@ -1760,7 +1780,9 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq, } if (nq) nq->budget++; - atomic_inc(&rdev->srq_count); + active_srqs = atomic_inc_return(&rdev->stats.res.srq_count); + if (active_srqs > rdev->stats.res.srq_watermark) + rdev->stats.res.srq_watermark = active_srqs; spin_lock_init(&srq->lock); return 0; @@ -1862,7 +1884,7 @@ static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev, int qp_attr_mask) { struct bnxt_re_qp *qp = rdev->gsi_ctx.gsi_sqp; - int rc = 0; + int rc; if (qp_attr_mask & IB_QP_STATE) { qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; @@ -2212,7 +2234,7 @@ static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp, u8 ip_version = 0; u16 vlan_id = 0xFFFF; void *buf; - int i, rc = 0; + int i, rc; memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr)); @@ -2250,7 +2272,7 @@ static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp, } is_eth = true; - is_vlan = (vlan_id && (vlan_id < 0x1000)) ? true : false; + is_vlan = vlan_id && (vlan_id < 0x1000); ib_ud_header_init(payload_size, !is_eth, is_eth, is_vlan, is_grh, ip_version, is_udp, 0, &qp->qp1_hdr); @@ -2787,7 +2809,6 @@ static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev, struct bnxt_qplib_swqe wqe; int rc = 0; - memset(&wqe, 0, sizeof(wqe)); while (wr) { /* House keeping */ memset(&wqe, 0, sizeof(wqe)); @@ -2886,7 +2907,7 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); ib_umem_release(cq->umem); - atomic_dec(&rdev->cq_count); + atomic_dec(&rdev->stats.res.cq_count); nq->budget--; kfree(cq->cql); return 0; @@ -2902,6 +2923,7 @@ int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, int cqe = attr->cqe; struct bnxt_qplib_nq *nq = NULL; unsigned int nq_alloc_cnt; + u32 active_cqs; if (attr->flags) return -EOPNOTSUPP; @@ -2970,7 +2992,9 @@ int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, cq->cq_period = cq->qplib_cq.period; nq->budget++; - atomic_inc(&rdev->cq_count); + active_cqs = atomic_inc_return(&rdev->stats.res.cq_count); + if (active_cqs > rdev->stats.res.cq_watermark) + rdev->stats.res.cq_watermark = active_cqs; spin_lock_init(&cq->cq_lock); if (udata) { @@ -3083,6 +3107,7 @@ int bnxt_re_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata) } cq->ib_cq.cqe = cq->resize_cqe; + atomic_inc(&rdev->stats.res.resize_count); return 0; @@ -3319,26 +3344,21 @@ static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp *gsi_qp, struct bnxt_re_dev *rdev = gsi_qp->rdev; struct bnxt_re_sqp_entries *sqp_entry = NULL; struct bnxt_re_qp *gsi_sqp = rdev->gsi_ctx.gsi_sqp; + dma_addr_t shrq_hdr_buf_map; + struct ib_sge s_sge[2] = {}; + struct ib_sge r_sge[2] = {}; struct bnxt_re_ah *gsi_sah; + struct ib_recv_wr rwr = {}; + dma_addr_t rq_hdr_buf_map; + struct ib_ud_wr udwr = {}; struct ib_send_wr *swr; - struct ib_ud_wr udwr; - struct ib_recv_wr rwr; + u32 skip_bytes = 0; int pkt_type = 0; - u32 tbl_idx; void *rq_hdr_buf; - dma_addr_t rq_hdr_buf_map; - dma_addr_t shrq_hdr_buf_map; u32 offset = 0; - u32 skip_bytes = 0; - struct ib_sge s_sge[2]; - struct ib_sge r_sge[2]; + u32 tbl_idx; int rc; - memset(&udwr, 0, sizeof(udwr)); - memset(&rwr, 0, sizeof(rwr)); - memset(&s_sge, 0, sizeof(s_sge)); - memset(&r_sge, 0, sizeof(r_sge)); - swr = &udwr.wr; tbl_idx = cqe->wr_id; @@ -3578,7 +3598,7 @@ static int send_phantom_wqe(struct bnxt_re_qp *qp) { struct bnxt_qplib_qp *lib_qp = &qp->qplib_qp; unsigned long flags; - int rc = 0; + int rc; spin_lock_irqsave(&qp->sq_lock, flags); @@ -3768,6 +3788,7 @@ struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags) struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); struct bnxt_re_dev *rdev = pd->rdev; struct bnxt_re_mr *mr; + u32 active_mrs; int rc; mr = kzalloc(sizeof(*mr), GFP_KERNEL); @@ -3795,7 +3816,9 @@ struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags) if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ | IB_ACCESS_REMOTE_ATOMIC)) mr->ib_mr.rkey = mr->ib_mr.lkey; - atomic_inc(&rdev->mr_count); + active_mrs = atomic_inc_return(&rdev->stats.res.mr_count); + if (active_mrs > rdev->stats.res.mr_watermark) + rdev->stats.res.mr_watermark = active_mrs; return &mr->ib_mr; @@ -3828,7 +3851,7 @@ int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) ib_umem_release(mr->ib_umem); kfree(mr); - atomic_dec(&rdev->mr_count); + atomic_dec(&rdev->stats.res.mr_count); return rc; } @@ -3858,6 +3881,7 @@ struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type, struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); struct bnxt_re_dev *rdev = pd->rdev; struct bnxt_re_mr *mr = NULL; + u32 active_mrs; int rc; if (type != IB_MR_TYPE_MEM_REG) { @@ -3896,7 +3920,9 @@ struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type, goto fail_mr; } - atomic_inc(&rdev->mr_count); + active_mrs = atomic_inc_return(&rdev->stats.res.mr_count); + if (active_mrs > rdev->stats.res.mr_watermark) + rdev->stats.res.mr_watermark = active_mrs; return &mr->ib_mr; fail_mr: @@ -3914,6 +3940,7 @@ struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type, struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); struct bnxt_re_dev *rdev = pd->rdev; struct bnxt_re_mw *mw; + u32 active_mws; int rc; mw = kzalloc(sizeof(*mw), GFP_KERNEL); @@ -3932,7 +3959,9 @@ struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type, } mw->ib_mw.rkey = mw->qplib_mw.rkey; - atomic_inc(&rdev->mw_count); + active_mws = atomic_inc_return(&rdev->stats.res.mw_count); + if (active_mws > rdev->stats.res.mw_watermark) + rdev->stats.res.mw_watermark = active_mws; return &mw->ib_mw; fail: @@ -3953,21 +3982,19 @@ int bnxt_re_dealloc_mw(struct ib_mw *ib_mw) } kfree(mw); - atomic_dec(&rdev->mw_count); + atomic_dec(&rdev->stats.res.mw_count); return rc; } -/* uverbs */ -struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, - u64 virt_addr, int mr_access_flags, - struct ib_udata *udata) +static struct ib_mr *__bnxt_re_user_reg_mr(struct ib_pd *ib_pd, u64 length, u64 virt_addr, + int mr_access_flags, struct ib_umem *umem) { struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); struct bnxt_re_dev *rdev = pd->rdev; - struct bnxt_re_mr *mr; - struct ib_umem *umem; unsigned long page_size; + struct bnxt_re_mr *mr; int umem_pgs, rc; + u32 active_mrs; if (length > BNXT_RE_MAX_MR_SIZE) { ibdev_err(&rdev->ibdev, "MR Size: %lld > Max supported:%lld\n", @@ -3975,6 +4002,12 @@ struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, return ERR_PTR(-ENOMEM); } + page_size = ib_umem_find_best_pgsz(umem, BNXT_RE_PAGE_SIZE_SUPPORTED, virt_addr); + if (!page_size) { + ibdev_err(&rdev->ibdev, "umem page size unsupported!"); + return ERR_PTR(-EINVAL); + } + mr = kzalloc(sizeof(*mr), GFP_KERNEL); if (!mr) return ERR_PTR(-ENOMEM); @@ -3986,45 +4019,33 @@ struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); if (rc) { - ibdev_err(&rdev->ibdev, "Failed to allocate MR"); + ibdev_err(&rdev->ibdev, "Failed to allocate MR rc = %d", rc); + rc = -EIO; goto free_mr; } /* The fixed portion of the rkey is the same as the lkey */ mr->ib_mr.rkey = mr->qplib_mr.rkey; - - umem = ib_umem_get(&rdev->ibdev, start, length, mr_access_flags); - if (IS_ERR(umem)) { - ibdev_err(&rdev->ibdev, "Failed to get umem"); - rc = -EFAULT; - goto free_mrw; - } mr->ib_umem = umem; - mr->qplib_mr.va = virt_addr; - page_size = ib_umem_find_best_pgsz( - umem, BNXT_RE_PAGE_SIZE_SUPPORTED, virt_addr); - if (!page_size) { - ibdev_err(&rdev->ibdev, "umem page size unsupported!"); - rc = -EFAULT; - goto free_umem; - } mr->qplib_mr.total_size = length; umem_pgs = ib_umem_num_dma_blocks(umem, page_size); rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, umem, umem_pgs, page_size); if (rc) { - ibdev_err(&rdev->ibdev, "Failed to register user MR"); - goto free_umem; + ibdev_err(&rdev->ibdev, "Failed to register user MR - rc = %d\n", rc); + rc = -EIO; + goto free_mrw; } mr->ib_mr.lkey = mr->qplib_mr.lkey; mr->ib_mr.rkey = mr->qplib_mr.lkey; - atomic_inc(&rdev->mr_count); + active_mrs = atomic_inc_return(&rdev->stats.res.mr_count); + if (active_mrs > rdev->stats.res.mr_watermark) + rdev->stats.res.mr_watermark = active_mrs; return &mr->ib_mr; -free_umem: - ib_umem_release(umem); + free_mrw: bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); free_mr: @@ -4032,6 +4053,48 @@ free_mr: return ERR_PTR(rc); } +struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, + u64 virt_addr, int mr_access_flags, + struct ib_udata *udata) +{ + struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); + struct bnxt_re_dev *rdev = pd->rdev; + struct ib_umem *umem; + struct ib_mr *ib_mr; + + umem = ib_umem_get(&rdev->ibdev, start, length, mr_access_flags); + if (IS_ERR(umem)) + return ERR_CAST(umem); + + ib_mr = __bnxt_re_user_reg_mr(ib_pd, length, virt_addr, mr_access_flags, umem); + if (IS_ERR(ib_mr)) + ib_umem_release(umem); + return ib_mr; +} + +struct ib_mr *bnxt_re_reg_user_mr_dmabuf(struct ib_pd *ib_pd, u64 start, + u64 length, u64 virt_addr, int fd, + int mr_access_flags, struct ib_udata *udata) +{ + struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); + struct bnxt_re_dev *rdev = pd->rdev; + struct ib_umem_dmabuf *umem_dmabuf; + struct ib_umem *umem; + struct ib_mr *ib_mr; + + umem_dmabuf = ib_umem_dmabuf_get_pinned(&rdev->ibdev, start, length, + fd, mr_access_flags); + if (IS_ERR(umem_dmabuf)) + return ERR_CAST(umem_dmabuf); + + umem = &umem_dmabuf->umem; + + ib_mr = __bnxt_re_user_reg_mr(ib_pd, length, virt_addr, mr_access_flags, umem); + if (IS_ERR(ib_mr)) + ib_umem_release(umem); + return ib_mr; +} + int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata) { struct ib_device *ibdev = ctx->device; @@ -4087,6 +4150,8 @@ int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata) goto cfail; } uctx->shpage_mmap = &entry->rdma_entry; + if (rdev->pacing.dbr_pacing) + resp.comp_mask |= BNXT_RE_UCNTX_CMASK_DBR_PACING_ENABLED; rc = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp))); if (rc) { @@ -4159,6 +4224,19 @@ int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma) case BNXT_RE_MMAP_SH_PAGE: ret = vm_insert_page(vma, vma->vm_start, virt_to_page(uctx->shpg)); break; + case BNXT_RE_MMAP_DBR_BAR: + pfn = bnxt_entry->mem_offset >> PAGE_SHIFT; + ret = rdma_user_mmap_io(ib_uctx, vma, pfn, PAGE_SIZE, + pgprot_noncached(vma->vm_page_prot), + rdma_entry); + break; + case BNXT_RE_MMAP_DBR_PAGE: + /* Driver doesn't expect write access for user space */ + if (vma->vm_flags & VM_WRITE) + return -EFAULT; + ret = vm_insert_page(vma, vma->vm_start, + virt_to_page((void *)bnxt_entry->mem_offset)); + break; default: ret = -EINVAL; break; @@ -4178,6 +4256,15 @@ void bnxt_re_mmap_free(struct rdma_user_mmap_entry *rdma_entry) kfree(bnxt_entry); } +static int UVERBS_HANDLER(BNXT_RE_METHOD_NOTIFY_DRV)(struct uverbs_attr_bundle *attrs) +{ + struct bnxt_re_ucontext *uctx; + + uctx = container_of(ib_uverbs_get_ucontext(attrs), struct bnxt_re_ucontext, ib_uctx); + bnxt_re_pacing_alert(uctx->rdev); + return 0; +} + static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle *attrs) { struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, BNXT_RE_ALLOC_PAGE_HANDLE); @@ -4190,7 +4277,7 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle * u64 mmap_offset; u32 length; u32 dpi; - u64 dbr; + u64 addr; int err; uctx = container_of(ib_uverbs_get_ucontext(attrs), struct bnxt_re_ucontext, ib_uctx); @@ -4212,19 +4299,30 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle * return -ENOMEM; length = PAGE_SIZE; dpi = uctx->wcdpi.dpi; - dbr = (u64)uctx->wcdpi.umdbr; + addr = (u64)uctx->wcdpi.umdbr; mmap_flag = BNXT_RE_MMAP_WC_DB; } else { return -EINVAL; } break; + case BNXT_RE_ALLOC_DBR_BAR_PAGE: + length = PAGE_SIZE; + addr = (u64)rdev->pacing.dbr_bar_addr; + mmap_flag = BNXT_RE_MMAP_DBR_BAR; + break; + + case BNXT_RE_ALLOC_DBR_PAGE: + length = PAGE_SIZE; + addr = (u64)rdev->pacing.dbr_page; + mmap_flag = BNXT_RE_MMAP_DBR_PAGE; + break; default: return -EOPNOTSUPP; } - entry = bnxt_re_mmap_entry_insert(uctx, dbr, mmap_flag, &mmap_offset); + entry = bnxt_re_mmap_entry_insert(uctx, addr, mmap_flag, &mmap_offset); if (!entry) return -ENOMEM; @@ -4264,6 +4362,9 @@ static int alloc_page_obj_cleanup(struct ib_uobject *uobject, uctx->wcdpi.dbr = NULL; } break; + case BNXT_RE_MMAP_DBR_BAR: + case BNXT_RE_MMAP_DBR_PAGE: + break; default: goto exit; } @@ -4301,7 +4402,13 @@ DECLARE_UVERBS_NAMED_OBJECT(BNXT_RE_OBJECT_ALLOC_PAGE, &UVERBS_METHOD(BNXT_RE_METHOD_ALLOC_PAGE), &UVERBS_METHOD(BNXT_RE_METHOD_DESTROY_PAGE)); +DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_NOTIFY_DRV); + +DECLARE_UVERBS_GLOBAL_METHODS(BNXT_RE_OBJECT_NOTIFY_DRV, + &UVERBS_METHOD(BNXT_RE_METHOD_NOTIFY_DRV)); + const struct uapi_definition bnxt_re_uapi_defs[] = { UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_ALLOC_PAGE), + UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_NOTIFY_DRV), {} }; diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h index 32d9e9d09791..84715b7e7a4e 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h @@ -146,6 +146,8 @@ enum bnxt_re_mmap_flag { BNXT_RE_MMAP_SH_PAGE, BNXT_RE_MMAP_UC_DB, BNXT_RE_MMAP_WC_DB, + BNXT_RE_MMAP_DBR_PAGE, + BNXT_RE_MMAP_DBR_BAR, }; struct bnxt_re_user_mmap_entry { @@ -227,6 +229,10 @@ int bnxt_re_dealloc_mw(struct ib_mw *mw); struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, u64 virt_addr, int mr_access_flags, struct ib_udata *udata); +struct ib_mr *bnxt_re_reg_user_mr_dmabuf(struct ib_pd *ib_pd, u64 start, + u64 length, u64 virt_addr, + int fd, int mr_access_flags, + struct ib_udata *udata); int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata); void bnxt_re_dealloc_ucontext(struct ib_ucontext *context); int bnxt_re_mmap(struct ib_ucontext *context, struct vm_area_struct *vma); diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c index 63e98e2d3596..c9066aade412 100644 --- a/drivers/infiniband/hw/bnxt_re/main.c +++ b/drivers/infiniband/hw/bnxt_re/main.c @@ -360,7 +360,7 @@ static struct bnxt_ulp_ops bnxt_re_ulp_ops = { static int bnxt_re_register_netdev(struct bnxt_re_dev *rdev) { struct bnxt_en_dev *en_dev; - int rc = 0; + int rc; en_dev = rdev->en_dev; @@ -395,10 +395,9 @@ static int bnxt_re_hwrm_qcfg(struct bnxt_re_dev *rdev, u32 *db_len, struct bnxt_en_dev *en_dev = rdev->en_dev; struct hwrm_func_qcfg_output resp = {0}; struct hwrm_func_qcfg_input req = {0}; - struct bnxt_fw_msg fw_msg; + struct bnxt_fw_msg fw_msg = {}; int rc; - memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr((void *)&req, HWRM_FUNC_QCFG); req.fid = cpu_to_le16(0xffff); bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, @@ -432,9 +431,219 @@ int bnxt_re_hwrm_qcaps(struct bnxt_re_dev *rdev) return rc; cctx->modes.db_push = le32_to_cpu(resp.flags) & FUNC_QCAPS_RESP_FLAGS_WCB_PUSH_MODE; + cctx->modes.dbr_pacing = + le32_to_cpu(resp.flags_ext2) & + FUNC_QCAPS_RESP_FLAGS_EXT2_DBR_PACING_EXT_SUPPORTED; + return 0; +} + +static int bnxt_re_hwrm_dbr_pacing_qcfg(struct bnxt_re_dev *rdev) +{ + struct hwrm_func_dbr_pacing_qcfg_output resp = {}; + struct hwrm_func_dbr_pacing_qcfg_input req = {}; + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct bnxt_qplib_chip_ctx *cctx; + struct bnxt_fw_msg fw_msg = {}; + int rc; + + cctx = rdev->chip_ctx; + bnxt_re_init_hwrm_hdr((void *)&req, HWRM_FUNC_DBR_PACING_QCFG); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = bnxt_send_msg(en_dev, &fw_msg); + if (rc) + return rc; + + if ((le32_to_cpu(resp.dbr_stat_db_fifo_reg) & + FUNC_DBR_PACING_QCFG_RESP_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK) == + FUNC_DBR_PACING_QCFG_RESP_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_GRC) + cctx->dbr_stat_db_fifo = + le32_to_cpu(resp.dbr_stat_db_fifo_reg) & + ~FUNC_DBR_PACING_QCFG_RESP_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK; + return 0; +} + +/* Update the pacing tunable parameters to the default values */ +static void bnxt_re_set_default_pacing_data(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data; + + pacing_data->do_pacing = rdev->pacing.dbr_def_do_pacing; + pacing_data->pacing_th = rdev->pacing.pacing_algo_th; + pacing_data->alarm_th = + pacing_data->pacing_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE; +} + +static void __wait_for_fifo_occupancy_below_th(struct bnxt_re_dev *rdev) +{ + u32 read_val, fifo_occup; + + /* loop shouldn't run infintely as the occupancy usually goes + * below pacing algo threshold as soon as pacing kicks in. + */ + while (1) { + read_val = readl(rdev->en_dev->bar0 + rdev->pacing.dbr_db_fifo_reg_off); + fifo_occup = BNXT_RE_MAX_FIFO_DEPTH - + ((read_val & BNXT_RE_DB_FIFO_ROOM_MASK) >> + BNXT_RE_DB_FIFO_ROOM_SHIFT); + /* Fifo occupancy cannot be greater the MAX FIFO depth */ + if (fifo_occup > BNXT_RE_MAX_FIFO_DEPTH) + break; + + if (fifo_occup < rdev->qplib_res.pacing_data->pacing_th) + break; + } +} + +static void bnxt_re_db_fifo_check(struct work_struct *work) +{ + struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev, + dbq_fifo_check_work); + struct bnxt_qplib_db_pacing_data *pacing_data; + u32 pacing_save; + + if (!mutex_trylock(&rdev->pacing.dbq_lock)) + return; + pacing_data = rdev->qplib_res.pacing_data; + pacing_save = rdev->pacing.do_pacing_save; + __wait_for_fifo_occupancy_below_th(rdev); + cancel_delayed_work_sync(&rdev->dbq_pacing_work); + if (pacing_save > rdev->pacing.dbr_def_do_pacing) { + /* Double the do_pacing value during the congestion */ + pacing_save = pacing_save << 1; + } else { + /* + * when a new congestion is detected increase the do_pacing + * by 8 times. And also increase the pacing_th by 4 times. The + * reason to increase pacing_th is to give more space for the + * queue to oscillate down without getting empty, but also more + * room for the queue to increase without causing another alarm. + */ + pacing_save = pacing_save << 3; + pacing_data->pacing_th = rdev->pacing.pacing_algo_th * 4; + } + + if (pacing_save > BNXT_RE_MAX_DBR_DO_PACING) + pacing_save = BNXT_RE_MAX_DBR_DO_PACING; + + pacing_data->do_pacing = pacing_save; + rdev->pacing.do_pacing_save = pacing_data->do_pacing; + pacing_data->alarm_th = + pacing_data->pacing_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE; + schedule_delayed_work(&rdev->dbq_pacing_work, + msecs_to_jiffies(rdev->pacing.dbq_pacing_time)); + rdev->stats.pacing.alerts++; + mutex_unlock(&rdev->pacing.dbq_lock); +} + +static void bnxt_re_pacing_timer_exp(struct work_struct *work) +{ + struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev, + dbq_pacing_work.work); + struct bnxt_qplib_db_pacing_data *pacing_data; + u32 read_val, fifo_occup; + + if (!mutex_trylock(&rdev->pacing.dbq_lock)) + return; + + pacing_data = rdev->qplib_res.pacing_data; + read_val = readl(rdev->en_dev->bar0 + rdev->pacing.dbr_db_fifo_reg_off); + fifo_occup = BNXT_RE_MAX_FIFO_DEPTH - + ((read_val & BNXT_RE_DB_FIFO_ROOM_MASK) >> + BNXT_RE_DB_FIFO_ROOM_SHIFT); + + if (fifo_occup > pacing_data->pacing_th) + goto restart_timer; + + /* + * Instead of immediately going back to the default do_pacing + * reduce it by 1/8 times and restart the timer. + */ + pacing_data->do_pacing = pacing_data->do_pacing - (pacing_data->do_pacing >> 3); + pacing_data->do_pacing = max_t(u32, rdev->pacing.dbr_def_do_pacing, pacing_data->do_pacing); + if (pacing_data->do_pacing <= rdev->pacing.dbr_def_do_pacing) { + bnxt_re_set_default_pacing_data(rdev); + rdev->stats.pacing.complete++; + goto dbq_unlock; + } + +restart_timer: + schedule_delayed_work(&rdev->dbq_pacing_work, + msecs_to_jiffies(rdev->pacing.dbq_pacing_time)); + rdev->stats.pacing.resched++; +dbq_unlock: + rdev->pacing.do_pacing_save = pacing_data->do_pacing; + mutex_unlock(&rdev->pacing.dbq_lock); +} + +void bnxt_re_pacing_alert(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_db_pacing_data *pacing_data; + + if (!rdev->pacing.dbr_pacing) + return; + mutex_lock(&rdev->pacing.dbq_lock); + pacing_data = rdev->qplib_res.pacing_data; + + /* + * Increase the alarm_th to max so that other user lib instances do not + * keep alerting the driver. + */ + pacing_data->alarm_th = BNXT_RE_MAX_FIFO_DEPTH; + pacing_data->do_pacing = BNXT_RE_MAX_DBR_DO_PACING; + cancel_work_sync(&rdev->dbq_fifo_check_work); + schedule_work(&rdev->dbq_fifo_check_work); + mutex_unlock(&rdev->pacing.dbq_lock); +} + +static int bnxt_re_initialize_dbr_pacing(struct bnxt_re_dev *rdev) +{ + if (bnxt_re_hwrm_dbr_pacing_qcfg(rdev)) + return -EIO; + + /* Allocate a page for app use */ + rdev->pacing.dbr_page = (void *)__get_free_page(GFP_KERNEL); + if (!rdev->pacing.dbr_page) + return -ENOMEM; + + memset((u8 *)rdev->pacing.dbr_page, 0, PAGE_SIZE); + rdev->qplib_res.pacing_data = (struct bnxt_qplib_db_pacing_data *)rdev->pacing.dbr_page; + + /* MAP HW window 2 for reading db fifo depth */ + writel(rdev->chip_ctx->dbr_stat_db_fifo & BNXT_GRC_BASE_MASK, + rdev->en_dev->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4); + rdev->pacing.dbr_db_fifo_reg_off = + (rdev->chip_ctx->dbr_stat_db_fifo & BNXT_GRC_OFFSET_MASK) + + BNXT_RE_GRC_FIFO_REG_BASE; + rdev->pacing.dbr_bar_addr = + pci_resource_start(rdev->qplib_res.pdev, 0) + rdev->pacing.dbr_db_fifo_reg_off; + + rdev->pacing.pacing_algo_th = BNXT_RE_PACING_ALGO_THRESHOLD; + rdev->pacing.dbq_pacing_time = BNXT_RE_DBR_PACING_TIME; + rdev->pacing.dbr_def_do_pacing = BNXT_RE_DBR_DO_PACING_NO_CONGESTION; + rdev->pacing.do_pacing_save = rdev->pacing.dbr_def_do_pacing; + rdev->qplib_res.pacing_data->fifo_max_depth = BNXT_RE_MAX_FIFO_DEPTH; + rdev->qplib_res.pacing_data->fifo_room_mask = BNXT_RE_DB_FIFO_ROOM_MASK; + rdev->qplib_res.pacing_data->fifo_room_shift = BNXT_RE_DB_FIFO_ROOM_SHIFT; + rdev->qplib_res.pacing_data->grc_reg_offset = rdev->pacing.dbr_db_fifo_reg_off; + bnxt_re_set_default_pacing_data(rdev); + /* Initialize worker for DBR Pacing */ + INIT_WORK(&rdev->dbq_fifo_check_work, bnxt_re_db_fifo_check); + INIT_DELAYED_WORK(&rdev->dbq_pacing_work, bnxt_re_pacing_timer_exp); return 0; } +static void bnxt_re_deinitialize_dbr_pacing(struct bnxt_re_dev *rdev) +{ + cancel_work_sync(&rdev->dbq_fifo_check_work); + cancel_delayed_work_sync(&rdev->dbq_pacing_work); + if (rdev->pacing.dbr_page) + free_page((u64)rdev->pacing.dbr_page); + + rdev->pacing.dbr_page = NULL; + rdev->pacing.dbr_pacing = false; +} + static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev, u16 fw_ring_id, int type) { @@ -652,6 +861,7 @@ static const struct ib_device_ops bnxt_re_dev_ops = { .query_qp = bnxt_re_query_qp, .query_srq = bnxt_re_query_srq, .reg_user_mr = bnxt_re_reg_user_mr, + .reg_user_mr_dmabuf = bnxt_re_reg_user_mr_dmabuf, .req_notify_cq = bnxt_re_req_notify_cq, .resize_cq = bnxt_re_resize_cq, INIT_RDMA_OBJ_SIZE(ib_ah, bnxt_re_ah, ib_ah), @@ -711,13 +921,14 @@ static struct bnxt_re_dev *bnxt_re_dev_add(struct bnxt_aux_priv *aux_priv, rdev->id = rdev->en_dev->pdev->devfn; INIT_LIST_HEAD(&rdev->qp_list); mutex_init(&rdev->qp_lock); - atomic_set(&rdev->qp_count, 0); - atomic_set(&rdev->cq_count, 0); - atomic_set(&rdev->srq_count, 0); - atomic_set(&rdev->mr_count, 0); - atomic_set(&rdev->mw_count, 0); - atomic_set(&rdev->ah_count, 0); - atomic_set(&rdev->pd_count, 0); + mutex_init(&rdev->pacing.dbq_lock); + atomic_set(&rdev->stats.res.qp_count, 0); + atomic_set(&rdev->stats.res.cq_count, 0); + atomic_set(&rdev->stats.res.srq_count, 0); + atomic_set(&rdev->stats.res.mr_count, 0); + atomic_set(&rdev->stats.res.mw_count, 0); + atomic_set(&rdev->stats.res.ah_count, 0); + atomic_set(&rdev->stats.res.pd_count, 0); rdev->cosq[0] = 0xFFFF; rdev->cosq[1] = 0xFFFF; @@ -759,7 +970,7 @@ static int bnxt_re_handle_unaffi_async_event(struct creq_func_event static int bnxt_re_handle_qp_async_event(struct creq_qp_event *qp_event, struct bnxt_re_qp *qp) { - struct ib_event event; + struct ib_event event = {}; unsigned int flags; if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR && @@ -769,7 +980,6 @@ static int bnxt_re_handle_qp_async_event(struct creq_qp_event *qp_event, bnxt_re_unlock_cqs(qp, flags); } - memset(&event, 0, sizeof(event)); if (qp->qplib_qp.srq) { event.device = &qp->rdev->ibdev; event.element.qp = &qp->ib_qp; @@ -937,13 +1147,12 @@ static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev) { struct bnxt_re_ring_attr rattr = {}; int num_vec_created = 0; - int rc = 0, i; + int rc, i; u8 type; /* Configure and allocate resources for qplib */ rdev->qplib_res.rcfw = &rdev->rcfw; - rc = bnxt_qplib_get_dev_attr(&rdev->rcfw, &rdev->dev_attr, - rdev->is_virtfn); + rc = bnxt_qplib_get_dev_attr(&rdev->rcfw, &rdev->dev_attr); if (rc) goto fail; @@ -1090,11 +1299,10 @@ static u32 bnxt_re_get_priority_mask(struct bnxt_re_dev *rdev) { u32 prio_map = 0, tmp_map = 0; struct net_device *netdev; - struct dcb_app app; + struct dcb_app app = {}; netdev = rdev->netdev; - memset(&app, 0, sizeof(app)); app.selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE; app.protocol = ETH_P_IBOE; tmp_map = dcb_ieee_getapp_mask(netdev, &app); @@ -1123,8 +1331,7 @@ static int bnxt_re_setup_qos(struct bnxt_re_dev *rdev) */ if ((prio_map == 0 && rdev->qplib_res.prio) || (prio_map != 0 && !rdev->qplib_res.prio)) { - rdev->qplib_res.prio = prio_map ? true : false; - + rdev->qplib_res.prio = prio_map; bnxt_re_update_gid(rdev); } @@ -1138,7 +1345,7 @@ static void bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev) struct hwrm_ver_get_input req = {}; struct bnxt_qplib_chip_ctx *cctx; struct bnxt_fw_msg fw_msg = {}; - int rc = 0; + int rc; bnxt_re_init_hwrm_hdr((void *)&req, HWRM_VER_GET); req.hwrm_intf_maj = HWRM_VERSION_MAJOR; @@ -1168,7 +1375,7 @@ static void bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev) static int bnxt_re_ib_init(struct bnxt_re_dev *rdev) { - int rc = 0; + int rc; u32 event; /* Register ib dev */ @@ -1214,8 +1421,11 @@ static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev) bnxt_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id, type); bnxt_qplib_free_rcfw_channel(&rdev->rcfw); } - if (test_and_clear_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags)) - rdev->num_msix = 0; + + rdev->num_msix = 0; + + if (rdev->pacing.dbr_pacing) + bnxt_re_deinitialize_dbr_pacing(rdev); bnxt_re_destroy_chip_ctx(rdev); if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags)) @@ -1234,15 +1444,14 @@ static void bnxt_re_worker(struct work_struct *work) static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 wqe_mode) { + struct bnxt_re_ring_attr rattr = {}; struct bnxt_qplib_creq_ctx *creq; - struct bnxt_re_ring_attr rattr; u32 db_offt; int vid; u8 type; int rc; /* Registered a new RoCE device instance to netdev */ - memset(&rattr, 0, sizeof(rattr)); rc = bnxt_re_register_netdev(rdev); if (rc) { ibdev_err(&rdev->ibdev, @@ -1271,7 +1480,6 @@ static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 wqe_mode) ibdev_dbg(&rdev->ibdev, "Got %d MSI-X vectors\n", rdev->en_dev->ulp_tbl->msix_requested); rdev->num_msix = rdev->en_dev->ulp_tbl->msix_requested; - set_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags); bnxt_re_query_hwrm_intf_version(rdev); @@ -1311,8 +1519,17 @@ static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 wqe_mode) goto free_ring; } - rc = bnxt_qplib_get_dev_attr(&rdev->rcfw, &rdev->dev_attr, - rdev->is_virtfn); + if (bnxt_qplib_dbr_pacing_en(rdev->chip_ctx)) { + rc = bnxt_re_initialize_dbr_pacing(rdev); + if (!rc) { + rdev->pacing.dbr_pacing = true; + } else { + ibdev_err(&rdev->ibdev, + "DBR pacing disabled with error : %d\n", rc); + rdev->pacing.dbr_pacing = false; + } + } + rc = bnxt_qplib_get_dev_attr(&rdev->rcfw, &rdev->dev_attr); if (rc) goto disable_rcfw; @@ -1400,7 +1617,7 @@ static int bnxt_re_add_device(struct auxiliary_device *adev, u8 wqe_mode) container_of(adev, struct bnxt_aux_priv, aux_dev); struct bnxt_en_dev *en_dev; struct bnxt_re_dev *rdev; - int rc = 0; + int rc; /* en_dev should never be NULL as long as adev and aux_dev are valid. */ en_dev = aux_priv->edev; @@ -1646,7 +1863,7 @@ static struct auxiliary_driver bnxt_re_driver = { static int __init bnxt_re_mod_init(void) { - int rc = 0; + int rc; pr_info("%s: %s", ROCE_DRV_MODULE_NAME, version); rc = auxiliary_driver_register(&bnxt_re_driver); diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c index a42555623aed..abbabea7f5fa 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c @@ -535,7 +535,7 @@ int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq, cqn_handler_t cqn_handler, srqn_handler_t srqn_handler) { - int rc = -1; + int rc; nq->pdev = pdev; nq->cqn_handler = cqn_handler; @@ -727,27 +727,30 @@ int bnxt_qplib_query_srq(struct bnxt_qplib_res *res, struct bnxt_qplib_rcfw *rcfw = res->rcfw; struct creq_query_srq_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; - struct bnxt_qplib_rcfw_sbuf *sbuf; + struct bnxt_qplib_rcfw_sbuf sbuf; struct creq_query_srq_resp_sb *sb; struct cmdq_query_srq req = {}; - int rc = 0; + int rc; bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req, CMDQ_BASE_OPCODE_QUERY_SRQ, sizeof(req)); /* Configure the request */ - sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb)); - if (!sbuf) + sbuf.size = ALIGN(sizeof(*sb), BNXT_QPLIB_CMDQE_UNITS); + sbuf.sb = dma_alloc_coherent(&rcfw->pdev->dev, sbuf.size, + &sbuf.dma_addr, GFP_KERNEL); + if (!sbuf.sb) return -ENOMEM; - req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS; + req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS; req.srq_cid = cpu_to_le32(srq->id); - sb = sbuf->sb; - bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, sbuf, sizeof(req), + sb = sbuf.sb; + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); srq->threshold = le16_to_cpu(sb->srq_limit); - bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf); + dma_free_coherent(&rcfw->pdev->dev, sbuf.size, + sbuf.sb, sbuf.dma_addr); return rc; } @@ -1365,24 +1368,26 @@ int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) struct bnxt_qplib_rcfw *rcfw = res->rcfw; struct creq_query_qp_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; - struct bnxt_qplib_rcfw_sbuf *sbuf; + struct bnxt_qplib_rcfw_sbuf sbuf; struct creq_query_qp_resp_sb *sb; struct cmdq_query_qp req = {}; u32 temp32[4]; - int i, rc = 0; + int i, rc; + + sbuf.size = ALIGN(sizeof(*sb), BNXT_QPLIB_CMDQE_UNITS); + sbuf.sb = dma_alloc_coherent(&rcfw->pdev->dev, sbuf.size, + &sbuf.dma_addr, GFP_KERNEL); + if (!sbuf.sb) + return -ENOMEM; + sb = sbuf.sb; bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req, CMDQ_BASE_OPCODE_QUERY_QP, sizeof(req)); - sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb)); - if (!sbuf) - return -ENOMEM; - sb = sbuf->sb; - req.qp_cid = cpu_to_le32(qp->id); - req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS; - bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, sbuf, sizeof(req), + req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS; + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc) @@ -1391,8 +1396,7 @@ int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) qp->state = sb->en_sqd_async_notify_state & CREQ_QUERY_QP_RESP_SB_STATE_MASK; qp->en_sqd_async_notify = sb->en_sqd_async_notify_state & - CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY ? - true : false; + CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY; qp->access = sb->access; qp->pkey_index = le16_to_cpu(sb->pkey); qp->qkey = le32_to_cpu(sb->qkey); @@ -1442,7 +1446,8 @@ int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) memcpy(qp->smac, sb->src_mac, 6); qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id); bail: - bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf); + dma_free_coherent(&rcfw->pdev->dev, sbuf.size, + sbuf.sb, sbuf.dma_addr); return rc; } diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c index bc3aea4592b9..c8c4017fe405 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c @@ -55,7 +55,7 @@ static void bnxt_qplib_service_creq(struct tasklet_struct *t); /** * bnxt_qplib_map_rc - map return type based on opcode - * @opcode - roce slow path opcode + * @opcode: roce slow path opcode * * case #1 * Firmware initiated error recovery is a safe state machine and @@ -98,8 +98,8 @@ static int bnxt_qplib_map_rc(u8 opcode) /** * bnxt_re_is_fw_stalled - Check firmware health - * @rcfw - rcfw channel instance of rdev - * @cookie - cookie to track the command + * @rcfw: rcfw channel instance of rdev + * @cookie: cookie to track the command * * If firmware has not responded any rcfw command within * rcfw->max_timeout, consider firmware as stalled. @@ -133,8 +133,8 @@ static int bnxt_re_is_fw_stalled(struct bnxt_qplib_rcfw *rcfw, /** * __wait_for_resp - Don't hold the cpu context and wait for response - * @rcfw - rcfw channel instance of rdev - * @cookie - cookie to track the command + * @rcfw: rcfw channel instance of rdev + * @cookie: cookie to track the command * * Wait for command completion in sleepable context. * @@ -179,8 +179,8 @@ static int __wait_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie) /** * __block_for_resp - hold the cpu context and wait for response - * @rcfw - rcfw channel instance of rdev - * @cookie - cookie to track the command + * @rcfw: rcfw channel instance of rdev + * @cookie: cookie to track the command * * This function will hold the cpu (non-sleepable context) and * wait for command completion. Maximum holding interval is 8 second. @@ -216,8 +216,8 @@ static int __block_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie) }; /* __send_message_no_waiter - get cookie and post the message. - * @rcfw - rcfw channel instance of rdev - * @msg - qplib message internal + * @rcfw: rcfw channel instance of rdev + * @msg: qplib message internal * * This function will just post and don't bother about completion. * Current design of this function is - @@ -335,7 +335,8 @@ static int __send_message(struct bnxt_qplib_rcfw *rcfw, cpu_to_le64(sbuf->dma_addr)); __set_cmdq_base_resp_size(msg->req, msg->req_sz, ALIGN(sbuf->size, - BNXT_QPLIB_CMDQE_UNITS)); + BNXT_QPLIB_CMDQE_UNITS) / + BNXT_QPLIB_CMDQE_UNITS); } preq = (u8 *)msg->req; @@ -373,8 +374,8 @@ static int __send_message(struct bnxt_qplib_rcfw *rcfw, /** * __poll_for_resp - self poll completion for rcfw command - * @rcfw - rcfw channel instance of rdev - * @cookie - cookie to track the command + * @rcfw: rcfw channel instance of rdev + * @cookie: cookie to track the command * * It works same as __wait_for_resp except this function will * do self polling in sort interval since interrupt is disabled. @@ -470,8 +471,8 @@ static void __destroy_timedout_ah(struct bnxt_qplib_rcfw *rcfw, /** * __bnxt_qplib_rcfw_send_message - qplib interface to send * and complete rcfw command. - * @rcfw - rcfw channel instance of rdev - * @msg - qplib message internal + * @rcfw: rcfw channel instance of rdev + * @msg: qplib message internal * * This function does not account shadow queue depth. It will send * all the command unconditionally as long as send queue is not full. @@ -487,7 +488,7 @@ static int __bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw, struct bnxt_qplib_crsqe *crsqe; unsigned long flags; u16 cookie; - int rc = 0; + int rc; u8 opcode; opcode = __get_cmdq_base_opcode(msg->req, msg->req_sz); @@ -533,8 +534,8 @@ static int __bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw, /** * bnxt_qplib_rcfw_send_message - qplib interface to send * and complete rcfw command. - * @rcfw - rcfw channel instance of rdev - * @msg - qplib message internal + * @rcfw: rcfw channel instance of rdev + * @msg: qplib message internal * * Driver interact with Firmware through rcfw channel/slow path in two ways. * a. Blocking rcfw command send. In this path, driver cannot hold @@ -1195,34 +1196,3 @@ int bnxt_qplib_enable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw, return 0; } - -struct bnxt_qplib_rcfw_sbuf *bnxt_qplib_rcfw_alloc_sbuf( - struct bnxt_qplib_rcfw *rcfw, - u32 size) -{ - struct bnxt_qplib_rcfw_sbuf *sbuf; - - sbuf = kzalloc(sizeof(*sbuf), GFP_KERNEL); - if (!sbuf) - return NULL; - - sbuf->size = size; - sbuf->sb = dma_alloc_coherent(&rcfw->pdev->dev, sbuf->size, - &sbuf->dma_addr, GFP_KERNEL); - if (!sbuf->sb) - goto bail; - - return sbuf; -bail: - kfree(sbuf); - return NULL; -} - -void bnxt_qplib_rcfw_free_sbuf(struct bnxt_qplib_rcfw *rcfw, - struct bnxt_qplib_rcfw_sbuf *sbuf) -{ - if (sbuf->sb) - dma_free_coherent(&rcfw->pdev->dev, sbuf->size, - sbuf->sb, sbuf->dma_addr); - kfree(sbuf); -} diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c index 739d942761d1..157db6b7e119 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_res.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c @@ -118,11 +118,11 @@ static int __alloc_pbl(struct bnxt_qplib_res *res, else pages = sginfo->npages; /* page ptr arrays */ - pbl->pg_arr = vmalloc(pages * sizeof(void *)); + pbl->pg_arr = vmalloc_array(pages, sizeof(void *)); if (!pbl->pg_arr) return -ENOMEM; - pbl->pg_map_arr = vmalloc(pages * sizeof(dma_addr_t)); + pbl->pg_map_arr = vmalloc_array(pages, sizeof(dma_addr_t)); if (!pbl->pg_map_arr) { vfree(pbl->pg_arr); pbl->pg_arr = NULL; @@ -385,7 +385,7 @@ static int bnxt_qplib_alloc_tqm_rings(struct bnxt_qplib_res *res, struct bnxt_qplib_hwq_attr hwq_attr = {}; struct bnxt_qplib_sg_info sginfo = {}; struct bnxt_qplib_tqm_ctx *tqmctx; - int rc = 0; + int rc; int i; tqmctx = &ctx->tqm_ctx; @@ -463,7 +463,7 @@ static void bnxt_qplib_map_tqm_pgtbl(struct bnxt_qplib_tqm_ctx *ctx) static int bnxt_qplib_setup_tqm_rings(struct bnxt_qplib_res *res, struct bnxt_qplib_ctx *ctx) { - int rc = 0; + int rc; rc = bnxt_qplib_alloc_tqm_rings(res, ctx); if (rc) @@ -501,7 +501,7 @@ int bnxt_qplib_alloc_ctx(struct bnxt_qplib_res *res, { struct bnxt_qplib_hwq_attr hwq_attr = {}; struct bnxt_qplib_sg_info sginfo = {}; - int rc = 0; + int rc; if (virt_fn || is_p5) goto stats_alloc; @@ -642,31 +642,44 @@ static void bnxt_qplib_init_sgid_tbl(struct bnxt_qplib_sgid_tbl *sgid_tbl, } /* PDs */ -int bnxt_qplib_alloc_pd(struct bnxt_qplib_pd_tbl *pdt, struct bnxt_qplib_pd *pd) +int bnxt_qplib_alloc_pd(struct bnxt_qplib_res *res, struct bnxt_qplib_pd *pd) { + struct bnxt_qplib_pd_tbl *pdt = &res->pd_tbl; u32 bit_num; + int rc = 0; + mutex_lock(&res->pd_tbl_lock); bit_num = find_first_bit(pdt->tbl, pdt->max); - if (bit_num == pdt->max) - return -ENOMEM; + if (bit_num == pdt->max) { + rc = -ENOMEM; + goto exit; + } /* Found unused PD */ clear_bit(bit_num, pdt->tbl); pd->id = bit_num; - return 0; +exit: + mutex_unlock(&res->pd_tbl_lock); + return rc; } int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res, struct bnxt_qplib_pd_tbl *pdt, struct bnxt_qplib_pd *pd) { + int rc = 0; + + mutex_lock(&res->pd_tbl_lock); if (test_and_set_bit(pd->id, pdt->tbl)) { dev_warn(&res->pdev->dev, "Freeing an unused PD? pdn = %d\n", pd->id); - return -EINVAL; + rc = -EINVAL; + goto exit; } pd->id = 0; - return 0; +exit: + mutex_unlock(&res->pd_tbl_lock); + return rc; } static void bnxt_qplib_free_pd_tbl(struct bnxt_qplib_pd_tbl *pdt) @@ -691,6 +704,7 @@ static int bnxt_qplib_alloc_pd_tbl(struct bnxt_qplib_res *res, pdt->max = max; memset((u8 *)pdt->tbl, 0xFF, bytes); + mutex_init(&res->pd_tbl_lock); return 0; } @@ -877,7 +891,7 @@ int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct pci_dev *pdev, struct net_device *netdev, struct bnxt_qplib_dev_attr *dev_attr) { - int rc = 0; + int rc; res->pdev = pdev; res->netdev = netdev; diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.h b/drivers/infiniband/hw/bnxt_re/qplib_res.h index d850a553821e..5949f004f785 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_res.h +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.h @@ -48,6 +48,7 @@ extern const struct bnxt_qplib_gid bnxt_qplib_gid_zero; struct bnxt_qplib_drv_modes { u8 wqe_mode; bool db_push; + bool dbr_pacing; }; struct bnxt_qplib_chip_ctx { @@ -58,6 +59,17 @@ struct bnxt_qplib_chip_ctx { u16 hwrm_cmd_max_timeout; struct bnxt_qplib_drv_modes modes; u64 hwrm_intf_ver; + u32 dbr_stat_db_fifo; +}; + +struct bnxt_qplib_db_pacing_data { + u32 do_pacing; + u32 pacing_th; + u32 alarm_th; + u32 fifo_max_depth; + u32 fifo_room_mask; + u32 fifo_room_shift; + u32 grc_reg_offset; }; #define BNXT_QPLIB_DBR_PF_DB_OFFSET 0x10000 @@ -265,12 +277,15 @@ struct bnxt_qplib_res { struct net_device *netdev; struct bnxt_qplib_rcfw *rcfw; struct bnxt_qplib_pd_tbl pd_tbl; + /* To protect the pd table bit map */ + struct mutex pd_tbl_lock; struct bnxt_qplib_sgid_tbl sgid_tbl; struct bnxt_qplib_dpi_tbl dpi_tbl; /* To protect the dpi table bit map */ struct mutex dpi_tbl_lock; bool prio; bool is_vf; + struct bnxt_qplib_db_pacing_data *pacing_data; }; static inline bool bnxt_qplib_is_chip_gen_p5(struct bnxt_qplib_chip_ctx *cctx) @@ -355,7 +370,7 @@ void bnxt_qplib_free_hwq(struct bnxt_qplib_res *res, struct bnxt_qplib_hwq *hwq); int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq, struct bnxt_qplib_hwq_attr *hwq_attr); -int bnxt_qplib_alloc_pd(struct bnxt_qplib_pd_tbl *pd_tbl, +int bnxt_qplib_alloc_pd(struct bnxt_qplib_res *res, struct bnxt_qplib_pd *pd); int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res, struct bnxt_qplib_pd_tbl *pd_tbl, @@ -467,4 +482,10 @@ static inline bool _is_ext_stats_supported(u16 dev_cap_flags) return dev_cap_flags & CREQ_QUERY_FUNC_RESP_SB_EXT_STATS; } + +static inline u8 bnxt_qplib_dbr_pacing_en(struct bnxt_qplib_chip_ctx *cctx) +{ + return cctx->modes.dbr_pacing; +} + #endif /* __BNXT_QPLIB_RES_H__ */ diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.c b/drivers/infiniband/hw/bnxt_re/qplib_sp.c index ab45f9d4bb02..a27b68515164 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_sp.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.c @@ -72,7 +72,7 @@ static void bnxt_qplib_query_version(struct bnxt_qplib_rcfw *rcfw, struct creq_query_version_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct cmdq_query_version req = {}; - int rc = 0; + int rc; bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req, CMDQ_BASE_OPCODE_QUERY_VERSION, @@ -89,31 +89,29 @@ static void bnxt_qplib_query_version(struct bnxt_qplib_rcfw *rcfw, } int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw, - struct bnxt_qplib_dev_attr *attr, bool vf) + struct bnxt_qplib_dev_attr *attr) { struct creq_query_func_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct creq_query_func_resp_sb *sb; - struct bnxt_qplib_rcfw_sbuf *sbuf; + struct bnxt_qplib_rcfw_sbuf sbuf; struct cmdq_query_func req = {}; u8 *tqm_alloc; - int i, rc = 0; + int i, rc; u32 temp; bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req, CMDQ_BASE_OPCODE_QUERY_FUNC, sizeof(req)); - sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb)); - if (!sbuf) { - dev_err(&rcfw->pdev->dev, - "SP: QUERY_FUNC alloc side buffer failed\n"); + sbuf.size = ALIGN(sizeof(*sb), BNXT_QPLIB_CMDQE_UNITS); + sbuf.sb = dma_alloc_coherent(&rcfw->pdev->dev, sbuf.size, + &sbuf.dma_addr, GFP_KERNEL); + if (!sbuf.sb) return -ENOMEM; - } - - sb = sbuf->sb; - req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS; - bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, sbuf, sizeof(req), + sb = sbuf.sb; + req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS; + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc) @@ -121,9 +119,8 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw, /* Extract the context from the side buffer */ attr->max_qp = le32_to_cpu(sb->max_qp); - /* max_qp value reported by FW for PF doesn't include the QP1 for PF */ - if (!vf) - attr->max_qp += 1; + /* max_qp value reported by FW doesn't include the QP1 */ + attr->max_qp += 1; attr->max_qp_rd_atom = sb->max_qp_rd_atom > BNXT_QPLIB_MAX_OUT_RD_ATOM ? BNXT_QPLIB_MAX_OUT_RD_ATOM : sb->max_qp_rd_atom; @@ -175,7 +172,8 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw, attr->is_atomic = bnxt_qplib_is_atomic_cap(rcfw); bail: - bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf); + dma_free_coherent(&rcfw->pdev->dev, sbuf.size, + sbuf.sb, sbuf.dma_addr); return rc; } @@ -186,7 +184,7 @@ int bnxt_qplib_set_func_resources(struct bnxt_qplib_res *res, struct creq_set_func_resources_resp resp = {}; struct cmdq_set_func_resources req = {}; struct bnxt_qplib_cmdqmsg msg = {}; - int rc = 0; + int rc; bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req, CMDQ_BASE_OPCODE_SET_FUNC_RESOURCES, @@ -718,23 +716,22 @@ int bnxt_qplib_get_roce_stats(struct bnxt_qplib_rcfw *rcfw, struct creq_query_roce_stats_resp_sb *sb; struct cmdq_query_roce_stats req = {}; struct bnxt_qplib_cmdqmsg msg = {}; - struct bnxt_qplib_rcfw_sbuf *sbuf; - int rc = 0; + struct bnxt_qplib_rcfw_sbuf sbuf; + int rc; bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req, CMDQ_BASE_OPCODE_QUERY_ROCE_STATS, sizeof(req)); - sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb)); - if (!sbuf) { - dev_err(&rcfw->pdev->dev, - "SP: QUERY_ROCE_STATS alloc side buffer failed\n"); + sbuf.size = ALIGN(sizeof(*sb), BNXT_QPLIB_CMDQE_UNITS); + sbuf.sb = dma_alloc_coherent(&rcfw->pdev->dev, sbuf.size, + &sbuf.dma_addr, GFP_KERNEL); + if (!sbuf.sb) return -ENOMEM; - } + sb = sbuf.sb; - sb = sbuf->sb; - req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS; - bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, sbuf, sizeof(req), + req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS; + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc) @@ -790,7 +787,8 @@ int bnxt_qplib_get_roce_stats(struct bnxt_qplib_rcfw *rcfw, } bail: - bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf); + dma_free_coherent(&rcfw->pdev->dev, sbuf.size, + sbuf.sb, sbuf.dma_addr); return rc; } @@ -801,49 +799,56 @@ int bnxt_qplib_qext_stat(struct bnxt_qplib_rcfw *rcfw, u32 fid, struct creq_query_roce_stats_ext_resp_sb *sb; struct cmdq_query_roce_stats_ext req = {}; struct bnxt_qplib_cmdqmsg msg = {}; - struct bnxt_qplib_rcfw_sbuf *sbuf; + struct bnxt_qplib_rcfw_sbuf sbuf; int rc; - sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb)); - if (!sbuf) { - dev_err(&rcfw->pdev->dev, - "SP: QUERY_ROCE_STATS_EXT alloc sb failed"); + sbuf.size = ALIGN(sizeof(*sb), BNXT_QPLIB_CMDQE_UNITS); + sbuf.sb = dma_alloc_coherent(&rcfw->pdev->dev, sbuf.size, + &sbuf.dma_addr, GFP_KERNEL); + if (!sbuf.sb) return -ENOMEM; - } + sb = sbuf.sb; bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req, CMDQ_QUERY_ROCE_STATS_EXT_OPCODE_QUERY_ROCE_STATS, sizeof(req)); - req.resp_size = ALIGN(sizeof(*sb), BNXT_QPLIB_CMDQE_UNITS); - req.resp_addr = cpu_to_le64(sbuf->dma_addr); + req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS; + req.resp_addr = cpu_to_le64(sbuf.dma_addr); req.function_id = cpu_to_le32(fid); req.flags = cpu_to_le16(CMDQ_QUERY_ROCE_STATS_EXT_FLAGS_FUNCTION_ID); - bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, sbuf, sizeof(req), + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc) goto bail; - sb = sbuf->sb; estat->tx_atomic_req = le64_to_cpu(sb->tx_atomic_req_pkts); estat->tx_read_req = le64_to_cpu(sb->tx_read_req_pkts); estat->tx_read_res = le64_to_cpu(sb->tx_read_res_pkts); estat->tx_write_req = le64_to_cpu(sb->tx_write_req_pkts); estat->tx_send_req = le64_to_cpu(sb->tx_send_req_pkts); + estat->tx_roce_pkts = le64_to_cpu(sb->tx_roce_pkts); + estat->tx_roce_bytes = le64_to_cpu(sb->tx_roce_bytes); estat->rx_atomic_req = le64_to_cpu(sb->rx_atomic_req_pkts); estat->rx_read_req = le64_to_cpu(sb->rx_read_req_pkts); estat->rx_read_res = le64_to_cpu(sb->rx_read_res_pkts); estat->rx_write_req = le64_to_cpu(sb->rx_write_req_pkts); estat->rx_send_req = le64_to_cpu(sb->rx_send_req_pkts); + estat->rx_roce_pkts = le64_to_cpu(sb->rx_roce_pkts); + estat->rx_roce_bytes = le64_to_cpu(sb->rx_roce_bytes); estat->rx_roce_good_pkts = le64_to_cpu(sb->rx_roce_good_pkts); estat->rx_roce_good_bytes = le64_to_cpu(sb->rx_roce_good_bytes); estat->rx_out_of_buffer = le64_to_cpu(sb->rx_out_of_buffer_pkts); estat->rx_out_of_sequence = le64_to_cpu(sb->rx_out_of_sequence_pkts); + estat->tx_cnp = le64_to_cpu(sb->tx_cnp_pkts); + estat->rx_cnp = le64_to_cpu(sb->rx_cnp_pkts); + estat->rx_ecn_marked = le64_to_cpu(sb->rx_ecn_marked_pkts); bail: - bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf); + dma_free_coherent(&rcfw->pdev->dev, sbuf.size, + sbuf.sb, sbuf.dma_addr); return rc; } diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.h b/drivers/infiniband/hw/bnxt_re/qplib_sp.h index 264ef3cedc45..d33c78b96217 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_sp.h +++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.h @@ -322,7 +322,7 @@ int bnxt_qplib_update_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, struct bnxt_qplib_gid *gid, u16 gid_idx, const u8 *smac); int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw, - struct bnxt_qplib_dev_attr *attr, bool vf); + struct bnxt_qplib_dev_attr *attr); int bnxt_qplib_set_func_resources(struct bnxt_qplib_res *res, struct bnxt_qplib_rcfw *rcfw, struct bnxt_qplib_ctx *ctx); |