diff options
Diffstat (limited to 'drivers/infiniband/hw/hfi1')
28 files changed, 561 insertions, 407 deletions
diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c index 4f057e8ffe50..e6a60fa59f2b 100644 --- a/drivers/infiniband/hw/hfi1/chip.c +++ b/drivers/infiniband/hw/hfi1/chip.c @@ -1083,6 +1083,7 @@ static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp, static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd); static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms); static void clear_rsm_rule(struct hfi1_devdata *dd, u8 rule_index); +static void update_xmit_counters(struct hfi1_pportdata *ppd, u16 link_width); /* * Error interrupt table entry. This is used as input to the interrupt @@ -6518,11 +6519,12 @@ static void _dc_start(struct hfi1_devdata *dd) if (!dd->dc_shutdown) return; - /* - * Take the 8051 out of reset, wait until 8051 is ready, and set host - * version bit. - */ - release_and_wait_ready_8051_firmware(dd); + /* Take the 8051 out of reset */ + write_csr(dd, DC_DC8051_CFG_RST, 0ull); + /* Wait until 8051 is ready */ + if (wait_fm_ready(dd, TIMEOUT_8051_START)) + dd_dev_err(dd, "%s: timeout starting 8051 firmware\n", + __func__); /* Take away reset for LCB and RX FPE (set in lcb_shutdown). */ write_csr(dd, DCC_CFG_RESET, 0x10); @@ -6904,6 +6906,32 @@ void handle_freeze(struct work_struct *work) /* no longer frozen */ } +/** + * update_xmit_counters - update PortXmitWait/PortVlXmitWait + * counters. + * @ppd: info of physical Hfi port + * @link_width: new link width after link up or downgrade + * + * Update the PortXmitWait and PortVlXmitWait counters after + * a link up or downgrade event to reflect a link width change. + */ +static void update_xmit_counters(struct hfi1_pportdata *ppd, u16 link_width) +{ + int i; + u16 tx_width; + u16 link_speed; + + tx_width = tx_link_width(link_width); + link_speed = get_link_speed(ppd->link_speed_active); + + /* + * There are C_VL_COUNT number of PortVLXmitWait counters. + * Adding 1 to C_VL_COUNT to include the PortXmitWait counter. + */ + for (i = 0; i < C_VL_COUNT + 1; i++) + get_xmit_wait_counters(ppd, tx_width, link_speed, i); +} + /* * Handle a link up interrupt from the 8051. * @@ -7525,18 +7553,29 @@ void handle_verify_cap(struct work_struct *work) set_link_state(ppd, HLS_GOING_UP); } -/* - * Apply the link width downgrade enabled policy against the current active - * link widths. +/** + * apply_link_downgrade_policy - Apply the link width downgrade enabled + * policy against the current active link widths. + * @ppd: info of physical Hfi port + * @refresh_widths: True indicates link downgrade event + * @return: True indicates a successful link downgrade. False indicates + * link downgrade event failed and the link will bounce back to + * default link width. * - * Called when the enabled policy changes or the active link widths change. + * Called when the enabled policy changes or the active link widths + * change. + * Refresh_widths indicates that a link downgrade occurred. The + * link_downgraded variable is set by refresh_widths and + * determines the success/failure of the policy application. */ -void apply_link_downgrade_policy(struct hfi1_pportdata *ppd, int refresh_widths) +bool apply_link_downgrade_policy(struct hfi1_pportdata *ppd, + bool refresh_widths) { int do_bounce = 0; int tries; u16 lwde; u16 tx, rx; + bool link_downgraded = refresh_widths; /* use the hls lock to avoid a race with actual link up */ tries = 0; @@ -7570,6 +7609,7 @@ retry: ppd->link_width_downgrade_rx_active == 0) { /* the 8051 reported a dead link as a downgrade */ dd_dev_err(ppd->dd, "Link downgrade is really a link down, ignoring\n"); + link_downgraded = false; } else if (lwde == 0) { /* downgrade is disabled */ @@ -7586,6 +7626,7 @@ retry: ppd->link_width_downgrade_tx_active, ppd->link_width_downgrade_rx_active); do_bounce = 1; + link_downgraded = false; } } else if ((lwde & ppd->link_width_downgrade_tx_active) == 0 || (lwde & ppd->link_width_downgrade_rx_active) == 0) { @@ -7597,6 +7638,7 @@ retry: lwde, ppd->link_width_downgrade_tx_active, ppd->link_width_downgrade_rx_active); do_bounce = 1; + link_downgraded = false; } done: @@ -7608,6 +7650,8 @@ done: set_link_state(ppd, HLS_DN_OFFLINE); start_link(ppd); } + + return link_downgraded; } /* @@ -7621,7 +7665,8 @@ void handle_link_downgrade(struct work_struct *work) link_downgrade_work); dd_dev_info(ppd->dd, "8051: Link width downgrade\n"); - apply_link_downgrade_policy(ppd, 1); + if (apply_link_downgrade_policy(ppd, true)) + update_xmit_counters(ppd, ppd->link_width_downgrade_tx_active); } static char *dcc_err_string(char *buf, int buf_len, u64 flags) @@ -8263,8 +8308,8 @@ static irqreturn_t sdma_interrupt(int irq, void *data) /* handle the interrupt(s) */ sdma_engine_interrupt(sde, status); } else { - dd_dev_err_ratelimited(dd, "SDMA engine %u interrupt, but no status bits set\n", - sde->this_idx); + dd_dev_info_ratelimited(dd, "SDMA engine %u interrupt, but no status bits set\n", + sde->this_idx); } return IRQ_HANDLED; } @@ -8564,23 +8609,27 @@ int write_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 data) } /* - * If the 8051 is in reset mode (dd->dc_shutdown == 1), this function - * will still continue executing. - * * Returns: * < 0 = Linux error, not able to get access * > 0 = 8051 command RETURN_CODE */ -static int _do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data, - u64 *out_data) +static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data, + u64 *out_data) { u64 reg, completed; int return_code; unsigned long timeout; - lockdep_assert_held(&dd->dc8051_lock); hfi1_cdbg(DC8051, "type %d, data 0x%012llx", type, in_data); + mutex_lock(&dd->dc8051_lock); + + /* We can't send any commands to the 8051 if it's in reset */ + if (dd->dc_shutdown) { + return_code = -ENODEV; + goto fail; + } + /* * If an 8051 host command timed out previously, then the 8051 is * stuck. @@ -8681,29 +8730,6 @@ static int _do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data, write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, 0); fail: - return return_code; -} - -/* - * Returns: - * < 0 = Linux error, not able to get access - * > 0 = 8051 command RETURN_CODE - */ -static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data, - u64 *out_data) -{ - int return_code; - - mutex_lock(&dd->dc8051_lock); - /* We can't send any commands to the 8051 if it's in reset */ - if (dd->dc_shutdown) { - return_code = -ENODEV; - goto fail; - } - - return_code = _do_8051_command(dd, type, in_data, out_data); - -fail: mutex_unlock(&dd->dc8051_lock); return return_code; } @@ -8713,17 +8739,16 @@ static int set_physical_link_state(struct hfi1_devdata *dd, u64 state) return do_8051_command(dd, HCMD_CHANGE_PHY_STATE, state, NULL); } -static int _load_8051_config(struct hfi1_devdata *dd, u8 field_id, - u8 lane_id, u32 config_data) +int load_8051_config(struct hfi1_devdata *dd, u8 field_id, + u8 lane_id, u32 config_data) { u64 data; int ret; - lockdep_assert_held(&dd->dc8051_lock); data = (u64)field_id << LOAD_DATA_FIELD_ID_SHIFT | (u64)lane_id << LOAD_DATA_LANE_ID_SHIFT | (u64)config_data << LOAD_DATA_DATA_SHIFT; - ret = _do_8051_command(dd, HCMD_LOAD_CONFIG_DATA, data, NULL); + ret = do_8051_command(dd, HCMD_LOAD_CONFIG_DATA, data, NULL); if (ret != HCMD_SUCCESS) { dd_dev_err(dd, "load 8051 config: field id %d, lane %d, err %d\n", @@ -8732,18 +8757,6 @@ static int _load_8051_config(struct hfi1_devdata *dd, u8 field_id, return ret; } -int load_8051_config(struct hfi1_devdata *dd, u8 field_id, - u8 lane_id, u32 config_data) -{ - int return_code; - - mutex_lock(&dd->dc8051_lock); - return_code = _load_8051_config(dd, field_id, lane_id, config_data); - mutex_unlock(&dd->dc8051_lock); - - return return_code; -} - /* * Read the 8051 firmware "registers". Use the RAM directly. Always * set the result, even on error. @@ -8859,14 +8872,13 @@ int write_host_interface_version(struct hfi1_devdata *dd, u8 version) u32 frame; u32 mask; - lockdep_assert_held(&dd->dc8051_lock); mask = (HOST_INTERFACE_VERSION_MASK << HOST_INTERFACE_VERSION_SHIFT); read_8051_config(dd, RESERVED_REGISTERS, GENERAL_CONFIG, &frame); /* Clear, then set field */ frame &= ~mask; frame |= ((u32)version << HOST_INTERFACE_VERSION_SHIFT); - return _load_8051_config(dd, RESERVED_REGISTERS, GENERAL_CONFIG, - frame); + return load_8051_config(dd, RESERVED_REGISTERS, GENERAL_CONFIG, + frame); } void read_misc_status(struct hfi1_devdata *dd, u8 *ver_major, u8 *ver_minor, @@ -9270,6 +9282,14 @@ static int set_local_link_attributes(struct hfi1_pportdata *ppd) if (ret != HCMD_SUCCESS) goto set_local_link_attributes_fail; + ret = write_host_interface_version(dd, HOST_INTERFACE_VERSION); + if (ret != HCMD_SUCCESS) { + dd_dev_err(dd, + "Failed to set host interface version, return 0x%x\n", + ret); + goto set_local_link_attributes_fail; + } + /* * DC supports continuous updates. */ @@ -10621,6 +10641,14 @@ int set_link_state(struct hfi1_pportdata *ppd, u32 state) add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK); handle_linkup_change(dd, 1); + + /* + * After link up, a new link width will have been set. + * Update the xmit counters with regards to the new + * link width. + */ + update_xmit_counters(ppd, ppd->link_width_active); + ppd->host_link_state = HLS_UP_INIT; update_statusp(ppd, IB_PORT_INIT); break; @@ -12984,7 +13012,14 @@ static void disable_intx(struct pci_dev *pdev) pci_intx(pdev, 0); } -static void clean_up_interrupts(struct hfi1_devdata *dd) +/** + * hfi1_clean_up_interrupts() - Free all IRQ resources + * @dd: valid device data data structure + * + * Free the MSI or INTx IRQs and assoicated PCI resources, + * if they have been allocated. + */ +void hfi1_clean_up_interrupts(struct hfi1_devdata *dd) { int i; @@ -13345,7 +13380,7 @@ static int set_up_interrupts(struct hfi1_devdata *dd) return 0; fail: - clean_up_interrupts(dd); + hfi1_clean_up_interrupts(dd); return ret; } @@ -14772,7 +14807,6 @@ void hfi1_start_cleanup(struct hfi1_devdata *dd) aspm_exit(dd); free_cntrs(dd); free_rcverr(dd); - clean_up_interrupts(dd); finish_chip_resources(dd); } @@ -14944,9 +14978,8 @@ struct hfi1_devdata *hfi1_init_dd(struct pci_dev *pdev, if (num_vls < HFI1_MIN_VLS_SUPPORTED || num_vls > HFI1_MAX_VLS_SUPPORTED) { - hfi1_early_err(&pdev->dev, - "Invalid num_vls %u, using %u VLs\n", - num_vls, HFI1_MAX_VLS_SUPPORTED); + dd_dev_err(dd, "Invalid num_vls %u, using %u VLs\n", + num_vls, HFI1_MAX_VLS_SUPPORTED); num_vls = HFI1_MAX_VLS_SUPPORTED; } ppd->vls_supported = num_vls; @@ -15229,7 +15262,7 @@ bail_free_rcverr: bail_free_cntrs: free_cntrs(dd); bail_clear_intr: - clean_up_interrupts(dd); + hfi1_clean_up_interrupts(dd); bail_cleanup: hfi1_pcie_ddcleanup(dd); bail_free: diff --git a/drivers/infiniband/hw/hfi1/chip.h b/drivers/infiniband/hw/hfi1/chip.h index 133e313feca4..c0d70f255050 100644 --- a/drivers/infiniband/hw/hfi1/chip.h +++ b/drivers/infiniband/hw/hfi1/chip.h @@ -508,6 +508,7 @@ #define DOWN_REMOTE_REASON_SHIFT 16 #define DOWN_REMOTE_REASON_MASK 0xff +#define HOST_INTERFACE_VERSION 1 #define HOST_INTERFACE_VERSION_SHIFT 16 #define HOST_INTERFACE_VERSION_MASK 0xff @@ -713,7 +714,6 @@ void read_misc_status(struct hfi1_devdata *dd, u8 *ver_major, u8 *ver_minor, u8 *ver_patch); int write_host_interface_version(struct hfi1_devdata *dd, u8 version); void read_guid(struct hfi1_devdata *dd); -int release_and_wait_ready_8051_firmware(struct hfi1_devdata *dd); int wait_fm_ready(struct hfi1_devdata *dd, u32 mstimeout); void set_link_down_reason(struct hfi1_pportdata *ppd, u8 lcl_reason, u8 neigh_reason, u8 rem_reason); @@ -736,8 +736,8 @@ int read_8051_config(struct hfi1_devdata *, u8, u8, u32 *); int start_link(struct hfi1_pportdata *ppd); int bringup_serdes(struct hfi1_pportdata *ppd); void set_intr_state(struct hfi1_devdata *dd, u32 enable); -void apply_link_downgrade_policy(struct hfi1_pportdata *ppd, - int refresh_widths); +bool apply_link_downgrade_policy(struct hfi1_pportdata *ppd, + bool refresh_widths); void update_usrhead(struct hfi1_ctxtdata *rcd, u32 hd, u32 updegr, u32 egrhd, u32 intr_adjust, u32 npkts); int stop_drain_data_vls(struct hfi1_devdata *dd); diff --git a/drivers/infiniband/hw/hfi1/debugfs.c b/drivers/infiniband/hw/hfi1/debugfs.c index 2e6e0c516041..852173bf05d0 100644 --- a/drivers/infiniband/hw/hfi1/debugfs.c +++ b/drivers/infiniband/hw/hfi1/debugfs.c @@ -1,5 +1,5 @@ /* - * Copyright(c) 2015-2017 Intel Corporation. + * Copyright(c) 2015-2018 Intel Corporation. * * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. @@ -1201,6 +1201,13 @@ static int _fault_stats_seq_show(struct seq_file *s, void *v) } hfi1_rcd_put(rcd); } + for_each_possible_cpu(j) { + struct hfi1_opcode_stats_perctx *sp = + per_cpu_ptr(dd->tx_opstats, j); + + n_packets += sp->stats[i].n_packets; + n_bytes += sp->stats[i].n_bytes; + } if (!n_packets && !n_bytes) return SEQ_SKIP; if (!ibd->fault_opcode->n_rxfaults[i] && diff --git a/drivers/infiniband/hw/hfi1/driver.c b/drivers/infiniband/hw/hfi1/driver.c index 4f65ac671044..addc68e83606 100644 --- a/drivers/infiniband/hw/hfi1/driver.c +++ b/drivers/infiniband/hw/hfi1/driver.c @@ -159,22 +159,6 @@ static int hfi1_caps_get(char *buffer, const struct kernel_param *kp) return scnprintf(buffer, PAGE_SIZE, "0x%lx", cap_mask); } -const char *get_unit_name(int unit) -{ - static char iname[16]; - - snprintf(iname, sizeof(iname), DRIVER_NAME "_%u", unit); - return iname; -} - -const char *get_card_name(struct rvt_dev_info *rdi) -{ - struct hfi1_ibdev *ibdev = container_of(rdi, struct hfi1_ibdev, rdi); - struct hfi1_devdata *dd = container_of(ibdev, - struct hfi1_devdata, verbs_dev); - return get_unit_name(dd->unit); -} - struct pci_dev *get_pci_dev(struct rvt_dev_info *rdi) { struct hfi1_ibdev *ibdev = container_of(rdi, struct hfi1_ibdev, rdi); @@ -272,7 +256,12 @@ static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd, u32 mlid_base; struct hfi1_ibport *ibp = rcd_to_iport(rcd); struct hfi1_devdata *dd = ppd->dd; - struct rvt_dev_info *rdi = &dd->verbs_dev.rdi; + struct hfi1_ibdev *verbs_dev = &dd->verbs_dev; + struct rvt_dev_info *rdi = &verbs_dev->rdi; + + if ((packet->rhf & RHF_DC_ERR) && + hfi1_dbg_fault_suppress_err(verbs_dev)) + return; if (packet->rhf & (RHF_VCRC_ERR | RHF_ICRC_ERR)) return; @@ -650,9 +639,10 @@ next: } } -static void process_rcv_qp_work(struct hfi1_ctxtdata *rcd) +static void process_rcv_qp_work(struct hfi1_packet *packet) { struct rvt_qp *qp, *nqp; + struct hfi1_ctxtdata *rcd = packet->rcd; /* * Iterate over all QPs waiting to respond. @@ -662,7 +652,8 @@ static void process_rcv_qp_work(struct hfi1_ctxtdata *rcd) list_del_init(&qp->rspwait); if (qp->r_flags & RVT_R_RSP_NAK) { qp->r_flags &= ~RVT_R_RSP_NAK; - hfi1_send_rc_ack(rcd, qp, 0); + packet->qp = qp; + hfi1_send_rc_ack(packet, 0); } if (qp->r_flags & RVT_R_RSP_SEND) { unsigned long flags; @@ -683,7 +674,7 @@ static noinline int max_packet_exceeded(struct hfi1_packet *packet, int thread) if (thread) { if ((packet->numpkt & (MAX_PKT_RECV_THREAD - 1)) == 0) /* allow defered processing */ - process_rcv_qp_work(packet->rcd); + process_rcv_qp_work(packet); cond_resched(); return RCV_PKT_OK; } else { @@ -825,7 +816,7 @@ int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread) last = RCV_PKT_DONE; process_rcv_update(last, &packet); } - process_rcv_qp_work(rcd); + process_rcv_qp_work(&packet); rcd->head = packet.rhqoff; bail: finish_packet(&packet); @@ -854,7 +845,7 @@ int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd, int thread) last = RCV_PKT_DONE; process_rcv_update(last, &packet); } - process_rcv_qp_work(rcd); + process_rcv_qp_work(&packet); rcd->head = packet.rhqoff; bail: finish_packet(&packet); @@ -1084,7 +1075,7 @@ int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread) process_rcv_update(last, &packet); } - process_rcv_qp_work(rcd); + process_rcv_qp_work(&packet); rcd->head = packet.rhqoff; bail: @@ -1454,8 +1445,8 @@ static int hfi1_setup_9B_packet(struct hfi1_packet *packet) packet->sc = hfi1_9B_get_sc5(hdr, packet->rhf); packet->pad = ib_bth_get_pad(packet->ohdr); packet->extra_byte = 0; - packet->fecn = ib_bth_get_fecn(packet->ohdr); - packet->becn = ib_bth_get_becn(packet->ohdr); + packet->pkey = ib_bth_get_pkey(packet->ohdr); + packet->migrated = ib_bth_is_migration(packet->ohdr); return 0; drop: @@ -1508,8 +1499,10 @@ static int hfi1_setup_bypass_packet(struct hfi1_packet *packet) /* Query commonly used fields from packet header */ packet->opcode = ib_bth_get_opcode(packet->ohdr); - packet->hlen = hdr_len_by_opcode[packet->opcode] + 8 + grh_len; - packet->payload = packet->ebuf + packet->hlen - (4 * sizeof(u32)); + /* hdr_len_by_opcode already has an IB LRH factored in */ + packet->hlen = hdr_len_by_opcode[packet->opcode] + + (LRH_16B_BYTES - LRH_9B_BYTES) + grh_len; + packet->payload = packet->ebuf + packet->hlen - LRH_16B_BYTES; packet->slid = hfi1_16B_get_slid(packet->hdr); packet->dlid = hfi1_16B_get_dlid(packet->hdr); if (unlikely(hfi1_is_16B_mcast(packet->dlid))) @@ -1520,8 +1513,8 @@ static int hfi1_setup_bypass_packet(struct hfi1_packet *packet) packet->sl = ibp->sc_to_sl[packet->sc]; packet->pad = hfi1_16B_bth_get_pad(packet->ohdr); packet->extra_byte = SIZE_OF_LT; - packet->fecn = hfi1_16B_get_fecn(packet->hdr); - packet->becn = hfi1_16B_get_becn(packet->hdr); + packet->pkey = hfi1_16B_get_pkey(packet->hdr); + packet->migrated = opa_bth_is_migration(packet->ohdr); if (hfi1_bypass_ingress_pkt_check(packet)) goto drop; @@ -1566,19 +1559,7 @@ int process_receive_ib(struct hfi1_packet *packet) if (hfi1_setup_9B_packet(packet)) return RHF_RCV_CONTINUE; - trace_hfi1_rcvhdr(packet->rcd->ppd->dd, - packet->rcd->ctxt, - rhf_err_flags(packet->rhf), - RHF_RCV_TYPE_IB, - packet->hlen, - packet->tlen, - packet->updegr, - rhf_egr_index(packet->rhf)); - - if (unlikely( - (hfi1_dbg_fault_suppress_err(&packet->rcd->dd->verbs_dev) && - (packet->rhf & RHF_DC_ERR)))) - return RHF_RCV_CONTINUE; + trace_hfi1_rcvhdr(packet); if (unlikely(rhf_err_flags(packet->rhf))) { handle_eflags(packet); @@ -1614,6 +1595,8 @@ int process_receive_bypass(struct hfi1_packet *packet) if (hfi1_setup_bypass_packet(packet)) return RHF_RCV_CONTINUE; + trace_hfi1_rcvhdr(packet); + if (unlikely(rhf_err_flags(packet->rhf))) { handle_eflags(packet); return RHF_RCV_CONTINUE; diff --git a/drivers/infiniband/hw/hfi1/file_ops.c b/drivers/infiniband/hw/hfi1/file_ops.c index 7750a9c38b06..41fafebe3b0d 100644 --- a/drivers/infiniband/hw/hfi1/file_ops.c +++ b/drivers/infiniband/hw/hfi1/file_ops.c @@ -74,7 +74,7 @@ static int hfi1_file_open(struct inode *inode, struct file *fp); static int hfi1_file_close(struct inode *inode, struct file *fp); static ssize_t hfi1_write_iter(struct kiocb *kiocb, struct iov_iter *from); -static unsigned int hfi1_poll(struct file *fp, struct poll_table_struct *pt); +static __poll_t hfi1_poll(struct file *fp, struct poll_table_struct *pt); static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma); static u64 kvirt_to_phys(void *addr); @@ -102,8 +102,8 @@ static int allocate_ctxt(struct hfi1_filedata *fd, struct hfi1_devdata *dd, struct hfi1_user_info *uinfo, struct hfi1_ctxtdata **cd); static void deallocate_ctxt(struct hfi1_ctxtdata *uctxt); -static unsigned int poll_urgent(struct file *fp, struct poll_table_struct *pt); -static unsigned int poll_next(struct file *fp, struct poll_table_struct *pt); +static __poll_t poll_urgent(struct file *fp, struct poll_table_struct *pt); +static __poll_t poll_next(struct file *fp, struct poll_table_struct *pt); static int user_event_ack(struct hfi1_ctxtdata *uctxt, u16 subctxt, unsigned long arg); static int set_ctxt_pkey(struct hfi1_ctxtdata *uctxt, unsigned long arg); @@ -196,9 +196,6 @@ static int hfi1_file_open(struct inode *inode, struct file *fp) if (!atomic_inc_not_zero(&dd->user_refcount)) return -ENXIO; - /* Just take a ref now. Not all opens result in a context assign */ - kobject_get(&dd->kobj); - /* The real work is performed later in assign_ctxt() */ fd = kzalloc(sizeof(*fd), GFP_KERNEL); @@ -208,6 +205,7 @@ static int hfi1_file_open(struct inode *inode, struct file *fp) fd->mm = current->mm; mmgrab(fd->mm); fd->dd = dd; + kobject_get(&fd->dd->kobj); fp->private_data = fd; } else { fp->private_data = NULL; @@ -607,20 +605,20 @@ static int vma_fault(struct vm_fault *vmf) return 0; } -static unsigned int hfi1_poll(struct file *fp, struct poll_table_struct *pt) +static __poll_t hfi1_poll(struct file *fp, struct poll_table_struct *pt) { struct hfi1_ctxtdata *uctxt; - unsigned pollflag; + __poll_t pollflag; uctxt = ((struct hfi1_filedata *)fp->private_data)->uctxt; if (!uctxt) - pollflag = POLLERR; + pollflag = EPOLLERR; else if (uctxt->poll_type == HFI1_POLL_TYPE_URGENT) pollflag = poll_urgent(fp, pt); else if (uctxt->poll_type == HFI1_POLL_TYPE_ANYRCV) pollflag = poll_next(fp, pt); else /* invalid */ - pollflag = POLLERR; + pollflag = EPOLLERR; return pollflag; } @@ -763,11 +761,11 @@ static int complete_subctxt(struct hfi1_filedata *fd) } if (ret) { - hfi1_rcd_put(fd->uctxt); - fd->uctxt = NULL; spin_lock_irqsave(&fd->dd->uctxt_lock, flags); __clear_bit(fd->subctxt, fd->uctxt->in_use_ctxts); spin_unlock_irqrestore(&fd->dd->uctxt_lock, flags); + hfi1_rcd_put(fd->uctxt); + fd->uctxt = NULL; } return ret; @@ -1425,19 +1423,19 @@ static int user_exp_rcv_invalid(struct hfi1_filedata *fd, unsigned long arg, return ret; } -static unsigned int poll_urgent(struct file *fp, +static __poll_t poll_urgent(struct file *fp, struct poll_table_struct *pt) { struct hfi1_filedata *fd = fp->private_data; struct hfi1_ctxtdata *uctxt = fd->uctxt; struct hfi1_devdata *dd = uctxt->dd; - unsigned pollflag; + __poll_t pollflag; poll_wait(fp, &uctxt->wait, pt); spin_lock_irq(&dd->uctxt_lock); if (uctxt->urgent != uctxt->urgent_poll) { - pollflag = POLLIN | POLLRDNORM; + pollflag = EPOLLIN | EPOLLRDNORM; uctxt->urgent_poll = uctxt->urgent; } else { pollflag = 0; @@ -1448,13 +1446,13 @@ static unsigned int poll_urgent(struct file *fp, return pollflag; } -static unsigned int poll_next(struct file *fp, +static __poll_t poll_next(struct file *fp, struct poll_table_struct *pt) { struct hfi1_filedata *fd = fp->private_data; struct hfi1_ctxtdata *uctxt = fd->uctxt; struct hfi1_devdata *dd = uctxt->dd; - unsigned pollflag; + __poll_t pollflag; poll_wait(fp, &uctxt->wait, pt); @@ -1464,7 +1462,7 @@ static unsigned int poll_next(struct file *fp, hfi1_rcvctrl(dd, HFI1_RCVCTRL_INTRAVAIL_ENB, uctxt); pollflag = 0; } else { - pollflag = POLLIN | POLLRDNORM; + pollflag = EPOLLIN | EPOLLRDNORM; } spin_unlock_irq(&dd->uctxt_lock); diff --git a/drivers/infiniband/hw/hfi1/firmware.c b/drivers/infiniband/hw/hfi1/firmware.c index 98868df78a7e..2b57ba70ddd6 100644 --- a/drivers/infiniband/hw/hfi1/firmware.c +++ b/drivers/infiniband/hw/hfi1/firmware.c @@ -68,7 +68,6 @@ #define ALT_FW_FABRIC_NAME "hfi1_fabric_d.fw" #define ALT_FW_SBUS_NAME "hfi1_sbus_d.fw" #define ALT_FW_PCIE_NAME "hfi1_pcie_d.fw" -#define HOST_INTERFACE_VERSION 1 MODULE_FIRMWARE(DEFAULT_FW_8051_NAME_ASIC); MODULE_FIRMWARE(DEFAULT_FW_FABRIC_NAME); @@ -976,46 +975,6 @@ int wait_fm_ready(struct hfi1_devdata *dd, u32 mstimeout) } /* - * Clear all reset bits, releasing the 8051. - * Wait for firmware to be ready to accept host requests. - * Then, set host version bit. - * - * This function executes even if the 8051 is in reset mode when - * dd->dc_shutdown == 1. - * - * Expects dd->dc8051_lock to be held. - */ -int release_and_wait_ready_8051_firmware(struct hfi1_devdata *dd) -{ - int ret; - - lockdep_assert_held(&dd->dc8051_lock); - /* clear all reset bits, releasing the 8051 */ - write_csr(dd, DC_DC8051_CFG_RST, 0ull); - - /* - * Wait for firmware to be ready to accept host - * requests. - */ - ret = wait_fm_ready(dd, TIMEOUT_8051_START); - if (ret) { - dd_dev_err(dd, "8051 start timeout, current FW state 0x%x\n", - get_firmware_state(dd)); - return ret; - } - - ret = write_host_interface_version(dd, HOST_INTERFACE_VERSION); - if (ret != HCMD_SUCCESS) { - dd_dev_err(dd, - "Failed to set host interface version, return 0x%x\n", - ret); - return -EIO; - } - - return 0; -} - -/* * Load the 8051 firmware. */ static int load_8051_firmware(struct hfi1_devdata *dd, @@ -1080,22 +1039,31 @@ static int load_8051_firmware(struct hfi1_devdata *dd, if (ret) return ret; + /* clear all reset bits, releasing the 8051 */ + write_csr(dd, DC_DC8051_CFG_RST, 0ull); + /* - * Clear all reset bits, releasing the 8051. * DC reset step 5. Wait for firmware to be ready to accept host * requests. - * Then, set host version bit. */ - mutex_lock(&dd->dc8051_lock); - ret = release_and_wait_ready_8051_firmware(dd); - mutex_unlock(&dd->dc8051_lock); - if (ret) - return ret; + ret = wait_fm_ready(dd, TIMEOUT_8051_START); + if (ret) { /* timed out */ + dd_dev_err(dd, "8051 start timeout, current state 0x%x\n", + get_firmware_state(dd)); + return -ETIMEDOUT; + } read_misc_status(dd, &ver_major, &ver_minor, &ver_patch); dd_dev_info(dd, "8051 firmware version %d.%d.%d\n", (int)ver_major, (int)ver_minor, (int)ver_patch); dd->dc8051_ver = dc8051_ver(ver_major, ver_minor, ver_patch); + ret = write_host_interface_version(dd, HOST_INTERFACE_VERSION); + if (ret != HCMD_SUCCESS) { + dd_dev_err(dd, + "Failed to set host interface version, return 0x%x\n", + ret); + return -EIO; + } return 0; } diff --git a/drivers/infiniband/hw/hfi1/hfi.h b/drivers/infiniband/hw/hfi1/hfi.h index 8ce9118d4a7f..90bc8c76d2ca 100644 --- a/drivers/infiniband/hw/hfi1/hfi.h +++ b/drivers/infiniband/hw/hfi1/hfi.h @@ -341,6 +341,7 @@ struct hfi1_packet { u32 slid; u16 tlen; s16 etail; + u16 pkey; u8 hlen; u8 numpkt; u8 rsize; @@ -351,8 +352,7 @@ struct hfi1_packet { u8 sc; u8 sl; u8 opcode; - bool becn; - bool fecn; + bool migrated; }; /* Packet types */ @@ -858,6 +858,13 @@ struct hfi1_pportdata { struct work_struct linkstate_active_work; /* Does this port need to prescan for FECNs */ bool cc_prescan; + /* + * Sample sendWaitCnt & sendWaitVlCnt during link transition + * and counter request. + */ + u64 port_vl_xmit_wait_last[C_VL_COUNT + 1]; + u16 prev_link_width; + u64 vl_xmit_flit_cnt[C_VL_COUNT + 1]; }; typedef int (*rhf_rcv_function_ptr)(struct hfi1_packet *packet); @@ -1623,7 +1630,7 @@ static int ingress_pkey_table_search(struct hfi1_pportdata *ppd, u16 pkey) * the 'error info' for this failure. */ static void ingress_pkey_table_fail(struct hfi1_pportdata *ppd, u16 pkey, - u16 slid) + u32 slid) { struct hfi1_devdata *dd = ppd->dd; @@ -1779,19 +1786,15 @@ void hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt, static inline bool process_ecn(struct rvt_qp *qp, struct hfi1_packet *pkt, bool do_cnp) { - struct ib_other_headers *ohdr = pkt->ohdr; - - u32 bth1; - bool becn = false; - bool fecn = false; + bool becn; + bool fecn; if (pkt->etype == RHF_RCV_TYPE_BYPASS) { fecn = hfi1_16B_get_fecn(pkt->hdr); becn = hfi1_16B_get_becn(pkt->hdr); } else { - bth1 = be32_to_cpu(ohdr->bth[1]); - fecn = bth1 & IB_FECN_SMASK; - becn = bth1 & IB_BECN_SMASK; + fecn = ib_bth_get_fecn(pkt->ohdr); + becn = ib_bth_get_becn(pkt->ohdr); } if (unlikely(fecn || becn)) { hfi1_process_ecn_slowpath(qp, pkt, do_cnp); @@ -1957,6 +1960,7 @@ void hfi1_verbs_unregister_sysfs(struct hfi1_devdata *dd); int qsfp_dump(struct hfi1_pportdata *ppd, char *buf, int len); int hfi1_pcie_init(struct pci_dev *pdev, const struct pci_device_id *ent); +void hfi1_clean_up_interrupts(struct hfi1_devdata *dd); void hfi1_pcie_cleanup(struct pci_dev *pdev); int hfi1_pcie_ddinit(struct hfi1_devdata *dd, struct pci_dev *pdev); void hfi1_pcie_ddcleanup(struct hfi1_devdata *); @@ -1971,8 +1975,6 @@ int get_platform_config_field(struct hfi1_devdata *dd, table_type, int table_index, int field_index, u32 *data, u32 len); -const char *get_unit_name(int unit); -const char *get_card_name(struct rvt_dev_info *rdi); struct pci_dev *get_pci_dev(struct rvt_dev_info *rdi); /* @@ -2122,39 +2124,42 @@ static inline u64 hfi1_pkt_base_sdma_integrity(struct hfi1_devdata *dd) #define dd_dev_emerg(dd, fmt, ...) \ dev_emerg(&(dd)->pcidev->dev, "%s: " fmt, \ - get_unit_name((dd)->unit), ##__VA_ARGS__) + rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), ##__VA_ARGS__) #define dd_dev_err(dd, fmt, ...) \ dev_err(&(dd)->pcidev->dev, "%s: " fmt, \ - get_unit_name((dd)->unit), ##__VA_ARGS__) + rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), ##__VA_ARGS__) #define dd_dev_err_ratelimited(dd, fmt, ...) \ dev_err_ratelimited(&(dd)->pcidev->dev, "%s: " fmt, \ - get_unit_name((dd)->unit), ##__VA_ARGS__) + rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), \ + ##__VA_ARGS__) #define dd_dev_warn(dd, fmt, ...) \ dev_warn(&(dd)->pcidev->dev, "%s: " fmt, \ - get_unit_name((dd)->unit), ##__VA_ARGS__) + rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), ##__VA_ARGS__) #define dd_dev_warn_ratelimited(dd, fmt, ...) \ dev_warn_ratelimited(&(dd)->pcidev->dev, "%s: " fmt, \ - get_unit_name((dd)->unit), ##__VA_ARGS__) + rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), \ + ##__VA_ARGS__) #define dd_dev_info(dd, fmt, ...) \ dev_info(&(dd)->pcidev->dev, "%s: " fmt, \ - get_unit_name((dd)->unit), ##__VA_ARGS__) + rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), ##__VA_ARGS__) #define dd_dev_info_ratelimited(dd, fmt, ...) \ dev_info_ratelimited(&(dd)->pcidev->dev, "%s: " fmt, \ - get_unit_name((dd)->unit), ##__VA_ARGS__) + rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), \ + ##__VA_ARGS__) #define dd_dev_dbg(dd, fmt, ...) \ dev_dbg(&(dd)->pcidev->dev, "%s: " fmt, \ - get_unit_name((dd)->unit), ##__VA_ARGS__) + rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), ##__VA_ARGS__) #define hfi1_dev_porterr(dd, port, fmt, ...) \ dev_err(&(dd)->pcidev->dev, "%s: port %u: " fmt, \ - get_unit_name((dd)->unit), (port), ##__VA_ARGS__) + rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), (port), ##__VA_ARGS__) /* * this is used for formatting hw error messages... @@ -2415,7 +2420,7 @@ static inline void hfi1_make_ib_hdr(struct ib_header *hdr, static inline void hfi1_make_16b_hdr(struct hfi1_16b_header *hdr, u32 slid, u32 dlid, u16 len, u16 pkey, - u8 becn, u8 fecn, u8 l4, + bool becn, bool fecn, u8 l4, u8 sc) { u32 lrh0 = 0; diff --git a/drivers/infiniband/hw/hfi1/init.c b/drivers/infiniband/hw/hfi1/init.c index 8e3b3e7d829a..33eba2356742 100644 --- a/drivers/infiniband/hw/hfi1/init.c +++ b/drivers/infiniband/hw/hfi1/init.c @@ -172,7 +172,7 @@ int hfi1_create_kctxts(struct hfi1_devdata *dd) u16 i; int ret; - dd->rcd = kzalloc_node(dd->num_rcv_contexts * sizeof(*dd->rcd), + dd->rcd = kcalloc_node(dd->num_rcv_contexts, sizeof(*dd->rcd), GFP_KERNEL, dd->node); if (!dd->rcd) return -ENOMEM; @@ -439,15 +439,16 @@ int hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, int numa, * The resulting value will be rounded down to the closest * multiple of dd->rcv_entries.group_size. */ - rcd->egrbufs.buffers = kzalloc_node( - rcd->egrbufs.count * sizeof(*rcd->egrbufs.buffers), - GFP_KERNEL, numa); + rcd->egrbufs.buffers = + kcalloc_node(rcd->egrbufs.count, + sizeof(*rcd->egrbufs.buffers), + GFP_KERNEL, numa); if (!rcd->egrbufs.buffers) goto bail; - rcd->egrbufs.rcvtids = kzalloc_node( - rcd->egrbufs.count * - sizeof(*rcd->egrbufs.rcvtids), - GFP_KERNEL, numa); + rcd->egrbufs.rcvtids = + kcalloc_node(rcd->egrbufs.count, + sizeof(*rcd->egrbufs.rcvtids), + GFP_KERNEL, numa); if (!rcd->egrbufs.rcvtids) goto bail; rcd->egrbufs.size = eager_buffer_size; @@ -637,6 +638,15 @@ void hfi1_init_pportdata(struct pci_dev *pdev, struct hfi1_pportdata *ppd, ppd->dd = dd; ppd->hw_pidx = hw_pidx; ppd->port = port; /* IB port number, not index */ + ppd->prev_link_width = LINK_WIDTH_DEFAULT; + /* + * There are C_VL_COUNT number of PortVLXmitWait counters. + * Adding 1 to C_VL_COUNT to include the PortXmitWait counter. + */ + for (i = 0; i < C_VL_COUNT + 1; i++) { + ppd->port_vl_xmit_wait_last[i] = 0; + ppd->vl_xmit_flit_cnt[i] = 0; + } default_pkey_idx = 1; @@ -1058,8 +1068,9 @@ static void shutdown_device(struct hfi1_devdata *dd) } dd->flags &= ~HFI1_INITTED; - /* mask interrupts, but not errors */ + /* mask and clean up interrupts, but not errors */ set_intr_state(dd, 0); + hfi1_clean_up_interrupts(dd); for (pidx = 0; pidx < dd->num_pports; ++pidx) { ppd = dd->pport + pidx; @@ -1218,6 +1229,7 @@ static void __hfi1_free_devdata(struct kobject *kobj) free_percpu(dd->rcv_limit); free_percpu(dd->send_schedule); free_percpu(dd->tx_opstats); + sdma_clean(dd, dd->num_sdma); rvt_dealloc_device(&dd->verbs_dev.rdi); } @@ -1272,6 +1284,8 @@ struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev, size_t extra) "Could not allocate unit ID: error %d\n", -ret); goto bail; } + rvt_set_ibdev_name(&dd->verbs_dev.rdi, "%s_%d", class_name(), dd->unit); + /* * Initialize all locks for the device. This needs to be as early as * possible so locks are usable. @@ -1702,6 +1716,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) dd_dev_err(dd, "Failed to create /dev devices: %d\n", -j); if (initfail || ret) { + hfi1_clean_up_interrupts(dd); stop_timers(dd); flush_workqueue(ib_wq); for (pidx = 0; pidx < dd->num_pports; ++pidx) { diff --git a/drivers/infiniband/hw/hfi1/iowait.h b/drivers/infiniband/hw/hfi1/iowait.h index 591697d85eed..3d9c32c7c340 100644 --- a/drivers/infiniband/hw/hfi1/iowait.h +++ b/drivers/infiniband/hw/hfi1/iowait.h @@ -371,4 +371,13 @@ static inline void iowait_starve_find_max(struct iowait *w, u8 *max, } } +/** + * iowait_packet_queued() - determine if a packet is already built + * @wait: the wait structure + */ +static inline bool iowait_packet_queued(struct iowait *wait) +{ + return !list_empty(&wait->tx_head); +} + #endif diff --git a/drivers/infiniband/hw/hfi1/mad.c b/drivers/infiniband/hw/hfi1/mad.c index cf8dba34fe30..e9962c65c68f 100644 --- a/drivers/infiniband/hw/hfi1/mad.c +++ b/drivers/infiniband/hw/hfi1/mad.c @@ -2649,6 +2649,79 @@ static void a0_portstatus(struct hfi1_pportdata *ppd, } } +/** + * tx_link_width - convert link width bitmask to integer + * value representing actual link width. + * @link_width: width of active link + * @return: return index of the bit set in link_width var + * + * The function convert and return the index of bit set + * that indicate the current link width. + */ +u16 tx_link_width(u16 link_width) +{ + int n = LINK_WIDTH_DEFAULT; + u16 tx_width = n; + + while (link_width && n) { + if (link_width & (1 << (n - 1))) { + tx_width = n; + break; + } + n--; + } + + return tx_width; +} + +/** + * get_xmit_wait_counters - Convert HFI 's SendWaitCnt/SendWaitVlCnt + * counter in unit of TXE cycle times to flit times. + * @ppd: info of physical Hfi port + * @link_width: width of active link + * @link_speed: speed of active link + * @vl: represent VL0-VL7, VL15 for PortVLXmitWait counters request + * and if vl value is C_VL_COUNT, it represent SendWaitCnt + * counter request + * @return: return SendWaitCnt/SendWaitVlCnt counter value per vl. + * + * Convert SendWaitCnt/SendWaitVlCnt counter from TXE cycle times to + * flit times. Call this function to samples these counters. This + * function will calculate for previous state transition and update + * current state at end of function using ppd->prev_link_width and + * ppd->port_vl_xmit_wait_last to port_vl_xmit_wait_curr and link_width. + */ +u64 get_xmit_wait_counters(struct hfi1_pportdata *ppd, + u16 link_width, u16 link_speed, int vl) +{ + u64 port_vl_xmit_wait_curr; + u64 delta_vl_xmit_wait; + u64 xmit_wait_val; + + if (vl > C_VL_COUNT) + return 0; + if (vl < C_VL_COUNT) + port_vl_xmit_wait_curr = + read_port_cntr(ppd, C_TX_WAIT_VL, vl); + else + port_vl_xmit_wait_curr = + read_port_cntr(ppd, C_TX_WAIT, CNTR_INVALID_VL); + + xmit_wait_val = + port_vl_xmit_wait_curr - + ppd->port_vl_xmit_wait_last[vl]; + delta_vl_xmit_wait = + convert_xmit_counter(xmit_wait_val, + ppd->prev_link_width, + link_speed); + + ppd->vl_xmit_flit_cnt[vl] += delta_vl_xmit_wait; + ppd->port_vl_xmit_wait_last[vl] = port_vl_xmit_wait_curr; + ppd->prev_link_width = link_width; + + return ppd->vl_xmit_flit_cnt[vl]; +} + static int pma_get_opa_portstatus(struct opa_pma_mad *pmp, struct ib_device *ibdev, u8 port, u32 *resp_len) @@ -2668,6 +2741,8 @@ static int pma_get_opa_portstatus(struct opa_pma_mad *pmp, struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); int vfi; u64 tmp, tmp2; + u16 link_width; + u16 link_speed; response_data_size = sizeof(struct opa_port_status_rsp) + num_vls * sizeof(struct _vls_pctrs); @@ -2711,8 +2786,16 @@ static int pma_get_opa_portstatus(struct opa_pma_mad *pmp, rsp->port_multicast_rcv_pkts = cpu_to_be64(read_dev_cntr(dd, C_DC_MC_RCV_PKTS, CNTR_INVALID_VL)); + /* + * Convert PortXmitWait counter from TXE cycle times + * to flit times. + */ + link_width = + tx_link_width(ppd->link_width_downgrade_tx_active); + link_speed = get_link_speed(ppd->link_speed_active); rsp->port_xmit_wait = - cpu_to_be64(read_port_cntr(ppd, C_TX_WAIT, CNTR_INVALID_VL)); + cpu_to_be64(get_xmit_wait_counters(ppd, link_width, + link_speed, C_VL_COUNT)); rsp->port_rcv_fecn = cpu_to_be64(read_dev_cntr(dd, C_DC_RCV_FCN, CNTR_INVALID_VL)); rsp->port_rcv_becn = @@ -2777,10 +2860,14 @@ static int pma_get_opa_portstatus(struct opa_pma_mad *pmp, rsp->vls[vfi].port_vl_xmit_pkts = cpu_to_be64(read_port_cntr(ppd, C_TX_PKT_VL, idx_from_vl(vl))); - + /* + * Convert PortVlXmitWait counter from TXE cycle + * times to flit times. + */ rsp->vls[vfi].port_vl_xmit_wait = - cpu_to_be64(read_port_cntr(ppd, C_TX_WAIT_VL, - idx_from_vl(vl))); + cpu_to_be64(get_xmit_wait_counters(ppd, link_width, + link_speed, + idx_from_vl(vl))); rsp->vls[vfi].port_vl_rcv_fecn = cpu_to_be64(read_dev_cntr(dd, C_DC_RCV_FCN_VL, @@ -2910,6 +2997,8 @@ static int pma_get_opa_datacounters(struct opa_pma_mad *pmp, unsigned long vl; u32 vl_select_mask; int vfi; + u16 link_width; + u16 link_speed; num_ports = be32_to_cpu(pmp->mad_hdr.attr_mod) >> 24; num_vls = hweight32(be32_to_cpu(req->vl_select_mask)); @@ -2959,8 +3048,16 @@ static int pma_get_opa_datacounters(struct opa_pma_mad *pmp, rsp->link_quality_indicator = cpu_to_be32((u32)lq); pma_get_opa_port_dctrs(ibdev, rsp); + /* + * Convert PortXmitWait counter from TXE + * cycle times to flit times. + */ + link_width = + tx_link_width(ppd->link_width_downgrade_tx_active); + link_speed = get_link_speed(ppd->link_speed_active); rsp->port_xmit_wait = - cpu_to_be64(read_port_cntr(ppd, C_TX_WAIT, CNTR_INVALID_VL)); + cpu_to_be64(get_xmit_wait_counters(ppd, link_width, + link_speed, C_VL_COUNT)); rsp->port_rcv_fecn = cpu_to_be64(read_dev_cntr(dd, C_DC_RCV_FCN, CNTR_INVALID_VL)); rsp->port_rcv_becn = @@ -2996,9 +3093,14 @@ static int pma_get_opa_datacounters(struct opa_pma_mad *pmp, cpu_to_be64(read_dev_cntr(dd, C_DC_RX_PKT_VL, idx_from_vl(vl))); + /* + * Convert PortVlXmitWait counter from TXE + * cycle times to flit times. + */ rsp->vls[vfi].port_vl_xmit_wait = - cpu_to_be64(read_port_cntr(ppd, C_TX_WAIT_VL, - idx_from_vl(vl))); + cpu_to_be64(get_xmit_wait_counters(ppd, link_width, + link_speed, + idx_from_vl(vl))); rsp->vls[vfi].port_vl_rcv_fecn = cpu_to_be64(read_dev_cntr(dd, C_DC_RCV_FCN_VL, @@ -3416,9 +3518,11 @@ static int pma_set_opa_portstatus(struct opa_pma_mad *pmp, if (counter_select & CS_PORT_MCAST_RCV_PKTS) write_dev_cntr(dd, C_DC_MC_RCV_PKTS, CNTR_INVALID_VL, 0); - if (counter_select & CS_PORT_XMIT_WAIT) + if (counter_select & CS_PORT_XMIT_WAIT) { write_port_cntr(ppd, C_TX_WAIT, CNTR_INVALID_VL, 0); - + ppd->port_vl_xmit_wait_last[C_VL_COUNT] = 0; + ppd->vl_xmit_flit_cnt[C_VL_COUNT] = 0; + } /* ignore cs_sw_portCongestion for HFIs */ if (counter_select & CS_PORT_RCV_FECN) @@ -3491,8 +3595,11 @@ static int pma_set_opa_portstatus(struct opa_pma_mad *pmp, if (counter_select & CS_PORT_RCV_PKTS) write_dev_cntr(dd, C_DC_RX_PKT_VL, idx_from_vl(vl), 0); - if (counter_select & CS_PORT_XMIT_WAIT) + if (counter_select & CS_PORT_XMIT_WAIT) { write_port_cntr(ppd, C_TX_WAIT_VL, idx_from_vl(vl), 0); + ppd->port_vl_xmit_wait_last[idx_from_vl(vl)] = 0; + ppd->vl_xmit_flit_cnt[idx_from_vl(vl)] = 0; + } /* sw_port_vl_congestion is 0 for HFIs */ if (counter_select & CS_PORT_RCV_FECN) @@ -4348,11 +4455,7 @@ static int opa_local_smp_check(struct hfi1_ibport *ibp, */ if (pkey == LIM_MGMT_P_KEY || pkey == FULL_MGMT_P_KEY) return 0; - /* - * On OPA devices it is okay to lose the upper 16 bits of LID as this - * information is obtained elsewhere. Mask off the upper 16 bits. - */ - ingress_pkey_table_fail(ppd, pkey, ib_lid_cpu16(0xFFFF & in_wc->slid)); + ingress_pkey_table_fail(ppd, pkey, in_wc->slid); return 1; } diff --git a/drivers/infiniband/hw/hfi1/mad.h b/drivers/infiniband/hw/hfi1/mad.h index c4938f3d97c8..2f48e6953629 100644 --- a/drivers/infiniband/hw/hfi1/mad.h +++ b/drivers/infiniband/hw/hfi1/mad.h @@ -180,6 +180,15 @@ struct opa_mad_notice_attr { #define OPA_VLARB_PREEMPT_MATRIX 3 #define IB_PMA_PORT_COUNTERS_CONG cpu_to_be16(0xFF00) +#define LINK_SPEED_25G 1 +#define LINK_SPEED_12_5G 2 +#define LINK_WIDTH_DEFAULT 4 +#define DECIMAL_FACTORING 1000 +/* + * The default link width is multiplied by 1000 + * to get accurate value after division. + */ +#define FACTOR_LINK_WIDTH (LINK_WIDTH_DEFAULT * DECIMAL_FACTORING) struct ib_pma_portcounters_cong { u8 reserved; @@ -429,5 +438,41 @@ struct sc2vlnt { void hfi1_event_pkey_change(struct hfi1_devdata *dd, u8 port); void hfi1_handle_trap_timer(struct timer_list *t); - +u16 tx_link_width(u16 link_width); +u64 get_xmit_wait_counters(struct hfi1_pportdata *ppd, u16 link_width, + u16 link_speed, int vl); +/** + * get_link_speed - determine whether 12.5G or 25G speed + * @link_speed: the speed of active link + * @return: Return 2 if link speed identified as 12.5G + * or return 1 if link speed is 25G. + * + * The function indirectly calculate required link speed + * value for convert_xmit_counter function. If the link + * speed is 25G, the function return as 1 as it is required + * by xmit counter conversion formula :-( 25G / link_speed). + * This conversion will provide value 1 if current + * link speed is 25G or 2 if 12.5G.This is done to avoid + * 12.5 float number conversion. + */ +static inline u16 get_link_speed(u16 link_speed) +{ + return (link_speed == 1) ? + LINK_SPEED_12_5G : LINK_SPEED_25G; +} + +/** + * convert_xmit_counter - calculate flit times for given xmit counter + * value + * @xmit_wait_val: current xmit counter value + * @link_width: width of active link + * @link_speed: speed of active link + * @return: return xmit counter value in flit times. + */ +static inline u64 convert_xmit_counter(u64 xmit_wait_val, u16 link_width, + u16 link_speed) +{ + return (xmit_wait_val * 2 * (FACTOR_LINK_WIDTH / link_width) + * link_speed) / DECIMAL_FACTORING; +} #endif /* _HFI1_MAD_H */ diff --git a/drivers/infiniband/hw/hfi1/mmu_rb.c b/drivers/infiniband/hw/hfi1/mmu_rb.c index e7b3ce123da6..70aceefe14d5 100644 --- a/drivers/infiniband/hw/hfi1/mmu_rb.c +++ b/drivers/infiniband/hw/hfi1/mmu_rb.c @@ -77,6 +77,7 @@ static void do_remove(struct mmu_rb_handler *handler, static void handle_remove(struct work_struct *work); static const struct mmu_notifier_ops mn_opts = { + .flags = MMU_INVALIDATE_DOES_NOT_BLOCK, .invalidate_range_start = mmu_notifier_range_start, }; diff --git a/drivers/infiniband/hw/hfi1/pcie.c b/drivers/infiniband/hw/hfi1/pcie.c index 8c7e7a60b715..83d66e862207 100644 --- a/drivers/infiniband/hw/hfi1/pcie.c +++ b/drivers/infiniband/hw/hfi1/pcie.c @@ -1034,6 +1034,7 @@ int do_pcie_gen3_transition(struct hfi1_devdata *dd) int do_retry, retry_count = 0; int intnum = 0; uint default_pset; + uint pset = pcie_pset; u16 target_vector, target_speed; u16 lnkctl2, vendor; u8 div; @@ -1201,16 +1202,16 @@ retry: * * Set Gen3EqPsetReqVec, leave other fields 0. */ - if (pcie_pset == UNSET_PSET) - pcie_pset = default_pset; - if (pcie_pset > 10) { /* valid range is 0-10, inclusive */ + if (pset == UNSET_PSET) + pset = default_pset; + if (pset > 10) { /* valid range is 0-10, inclusive */ dd_dev_err(dd, "%s: Invalid Eq Pset %u, setting to %d\n", - __func__, pcie_pset, default_pset); - pcie_pset = default_pset; + __func__, pset, default_pset); + pset = default_pset; } - dd_dev_info(dd, "%s: using EQ Pset %u\n", __func__, pcie_pset); + dd_dev_info(dd, "%s: using EQ Pset %u\n", __func__, pset); pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL106, - ((1 << pcie_pset) << + ((1 << pset) << PCIE_CFG_REG_PL106_GEN3_EQ_PSET_REQ_VEC_SHIFT) | PCIE_CFG_REG_PL106_GEN3_EQ_EVAL2MS_DISABLE_SMASK | PCIE_CFG_REG_PL106_GEN3_EQ_PHASE23_EXIT_MODE_SMASK); @@ -1240,10 +1241,10 @@ retry: /* apply static CTLE tunings */ u8 pcie_dc, pcie_lf, pcie_hf, pcie_bw; - pcie_dc = ctle_tunings[pcie_pset][0]; - pcie_lf = ctle_tunings[pcie_pset][1]; - pcie_hf = ctle_tunings[pcie_pset][2]; - pcie_bw = ctle_tunings[pcie_pset][3]; + pcie_dc = ctle_tunings[pset][0]; + pcie_lf = ctle_tunings[pset][1]; + pcie_hf = ctle_tunings[pset][2]; + pcie_bw = ctle_tunings[pset][3]; write_gasket_interrupt(dd, intnum++, 0x0026, 0x0200 | pcie_dc); write_gasket_interrupt(dd, intnum++, 0x0026, 0x0100 | pcie_lf); write_gasket_interrupt(dd, intnum++, 0x0026, 0x0000 | pcie_hf); diff --git a/drivers/infiniband/hw/hfi1/pio.c b/drivers/infiniband/hw/hfi1/pio.c index 4c1198bc5e70..40dac4d16eb8 100644 --- a/drivers/infiniband/hw/hfi1/pio.c +++ b/drivers/infiniband/hw/hfi1/pio.c @@ -455,8 +455,8 @@ int init_send_contexts(struct hfi1_devdata *dd) dd->hw_to_sw = kmalloc_array(TXE_NUM_CONTEXTS, sizeof(u8), GFP_KERNEL); dd->send_contexts = kcalloc(dd->num_send_contexts, - sizeof(struct send_context_info), - GFP_KERNEL); + sizeof(struct send_context_info), + GFP_KERNEL); if (!dd->send_contexts || !dd->hw_to_sw) { kfree(dd->hw_to_sw); kfree(dd->send_contexts); @@ -856,8 +856,9 @@ struct send_context *sc_alloc(struct hfi1_devdata *dd, int type, * so head == tail can mean empty. */ sc->sr_size = sci->credits + 1; - sc->sr = kzalloc_node(sizeof(union pio_shadow_ring) * - sc->sr_size, GFP_KERNEL, numa); + sc->sr = kcalloc_node(sc->sr_size, + sizeof(union pio_shadow_ring), + GFP_KERNEL, numa); if (!sc->sr) { sc_free(sc); return NULL; @@ -1958,9 +1959,9 @@ int init_pervl_scs(struct hfi1_devdata *dd) hfi1_init_ctxt(dd->vld[15].sc); dd->vld[15].mtu = enum_to_mtu(OPA_MTU_2048); - dd->kernel_send_context = kzalloc_node(dd->num_send_contexts * - sizeof(struct send_context *), - GFP_KERNEL, dd->node); + dd->kernel_send_context = kcalloc_node(dd->num_send_contexts, + sizeof(struct send_context *), + GFP_KERNEL, dd->node); if (!dd->kernel_send_context) goto freesc15; diff --git a/drivers/infiniband/hw/hfi1/qp.c b/drivers/infiniband/hw/hfi1/qp.c index 4b01ccd895b4..d30dd1a5b0a6 100644 --- a/drivers/infiniband/hw/hfi1/qp.c +++ b/drivers/infiniband/hw/hfi1/qp.c @@ -556,6 +556,8 @@ void qp_iter_print(struct seq_file *s, struct rvt_qp_iter *iter) struct sdma_engine *sde; struct send_context *send_context; struct rvt_ack_entry *e = NULL; + struct rvt_srq *srq = qp->ibqp.srq ? + ibsrq_to_rvtsrq(qp->ibqp.srq) : NULL; sde = qp_to_sdma_engine(qp, priv->s_sc); wqe = rvt_get_swqe_ptr(qp, qp->s_last); @@ -563,7 +565,7 @@ void qp_iter_print(struct seq_file *s, struct rvt_qp_iter *iter) if (qp->s_ack_queue) e = &qp->s_ack_queue[qp->s_tail_ack_queue]; seq_printf(s, - "N %d %s QP %x R %u %s %u %u %u f=%x %u %u %u %u %u %u SPSN %x %x %x %x %x RPSN %x S(%u %u %u %u %u %u %u) R(%u %u %u) RQP %x LID %x SL %u MTU %u %u %u %u %u SDE %p,%u SC %p,%u SCQ %u %u PID %d OS %x %x E %x %x %x\n", + "N %d %s QP %x R %u %s %u %u f=%x %u %u %u %u %u %u SPSN %x %x %x %x %x RPSN %x S(%u %u %u %u %u %u %u) R(%u %u %u) RQP %x LID %x SL %u MTU %u %u %u %u %u SDE %p,%u SC %p,%u SCQ %u %u PID %d OS %x %x E %x %x %x RNR %d %s %d\n", iter->n, qp_idle(qp) ? "I" : "B", qp->ibqp.qp_num, @@ -571,7 +573,6 @@ void qp_iter_print(struct seq_file *s, struct rvt_qp_iter *iter) qp_type_str[qp->ibqp.qp_type], qp->state, wqe ? wqe->wr.opcode : 0, - qp->s_hdrwords, qp->s_flags, iowait_sdma_pending(&priv->s_iowait), iowait_pio_pending(&priv->s_iowait), @@ -610,7 +611,11 @@ void qp_iter_print(struct seq_file *s, struct rvt_qp_iter *iter) /* ack queue information */ e ? e->opcode : 0, e ? e->psn : 0, - e ? e->lpsn : 0); + e ? e->lpsn : 0, + qp->r_min_rnr_timer, + srq ? "SRQ" : "RQ", + srq ? srq->rq.size : qp->r_rq.size + ); } void *qp_priv_alloc(struct rvt_dev_info *rdi, struct rvt_qp *qp) @@ -789,7 +794,6 @@ void notify_error_qp(struct rvt_qp *qp) } if (!(qp->s_flags & RVT_S_BUSY)) { - qp->s_hdrwords = 0; if (qp->s_rdma_mr) { rvt_put_mr(qp->s_rdma_mr); qp->s_rdma_mr = NULL; diff --git a/drivers/infiniband/hw/hfi1/qp.h b/drivers/infiniband/hw/hfi1/qp.h index c06d2f8348e0..b2d4cba8d15b 100644 --- a/drivers/infiniband/hw/hfi1/qp.h +++ b/drivers/infiniband/hw/hfi1/qp.h @@ -51,12 +51,25 @@ #include <rdma/rdmavt_qp.h> #include "verbs.h" #include "sdma.h" +#include "verbs_txreq.h" extern unsigned int hfi1_qp_table_size; extern const struct rvt_operation_params hfi1_post_parms[]; /* + * Send if not busy or waiting for I/O and either + * a RC response is pending or we can process send work requests. + */ +static inline int hfi1_send_ok(struct rvt_qp *qp) +{ + return !(qp->s_flags & (RVT_S_BUSY | RVT_S_ANY_WAIT_IO)) && + (verbs_txreq_queued(qp) || + (qp->s_flags & RVT_S_RESP_PENDING) || + !(qp->s_flags & RVT_S_ANY_WAIT_SEND)); +} + +/* * free_ahg - clear ahg from QP */ static inline void clear_ahg(struct rvt_qp *qp) diff --git a/drivers/infiniband/hw/hfi1/rc.c b/drivers/infiniband/hw/hfi1/rc.c index af5f7936f7e5..da58046a02ea 100644 --- a/drivers/infiniband/hw/hfi1/rc.c +++ b/drivers/infiniband/hw/hfi1/rc.c @@ -226,12 +226,10 @@ normal: bth2 = mask_psn(qp->s_ack_psn); } qp->s_rdma_ack_cnt++; - qp->s_hdrwords = hwords; ps->s_txreq->sde = priv->s_sde; ps->s_txreq->s_cur_size = len; + ps->s_txreq->hdr_dwords = hwords; hfi1_make_ruc_header(qp, ohdr, bth0, bth2, middle, ps); - /* pbc */ - ps->s_txreq->hdr_dwords = qp->s_hdrwords + 2; return 1; bail: @@ -302,7 +300,6 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND)) goto bail; /* We are in the error state, flush the work request. */ - smp_read_barrier_depends(); /* see post_one_send() */ if (qp->s_last == READ_ONCE(qp->s_head)) goto bail; /* If DMAs are in progress, we can't flush immediately. */ @@ -346,7 +343,6 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) newreq = 0; if (qp->s_cur == qp->s_tail) { /* Check if send work queue is empty. */ - smp_read_barrier_depends(); /* see post_one_send() */ if (qp->s_tail == READ_ONCE(qp->s_head)) { clear_ahg(qp); goto bail; @@ -387,7 +383,6 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) : IB_WC_SUCCESS); if (local_ops) atomic_dec(&qp->local_ops_pending); - qp->s_hdrwords = 0; goto done_free_tx; } @@ -690,7 +685,7 @@ no_flow_control: bth2 |= IB_BTH_REQ_ACK; } qp->s_len -= len; - qp->s_hdrwords = hwords; + ps->s_txreq->hdr_dwords = hwords; ps->s_txreq->sde = priv->s_sde; ps->s_txreq->ss = ss; ps->s_txreq->s_cur_size = len; @@ -701,8 +696,6 @@ no_flow_control: bth2, middle, ps); - /* pbc */ - ps->s_txreq->hdr_dwords = qp->s_hdrwords + 2; return 1; done_free_tx: @@ -716,7 +709,6 @@ bail: bail_no_tx: ps->s_txreq = NULL; qp->s_flags &= ~RVT_S_BUSY; - qp->s_hdrwords = 0; return 0; } @@ -736,14 +728,16 @@ static inline void hfi1_make_bth_aeth(struct rvt_qp *qp, ohdr->bth[2] = cpu_to_be32(mask_psn(qp->r_ack_psn)); } -static inline void hfi1_queue_rc_ack(struct rvt_qp *qp, bool is_fecn) +static inline void hfi1_queue_rc_ack(struct hfi1_packet *packet, bool is_fecn) { - struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); + struct rvt_qp *qp = packet->qp; + struct hfi1_ibport *ibp; unsigned long flags; spin_lock_irqsave(&qp->s_lock, flags); if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) goto unlock; + ibp = rcd_to_iport(packet->rcd); this_cpu_inc(*ibp->rvp.rc_qacks); qp->s_flags |= RVT_S_ACK_PENDING | RVT_S_RESP_PENDING; qp->s_nak_state = qp->r_nak_state; @@ -757,13 +751,14 @@ unlock: spin_unlock_irqrestore(&qp->s_lock, flags); } -static inline void hfi1_make_rc_ack_9B(struct rvt_qp *qp, +static inline void hfi1_make_rc_ack_9B(struct hfi1_packet *packet, struct hfi1_opa_header *opa_hdr, u8 sc5, bool is_fecn, u64 *pbc_flags, u32 *hwords, u32 *nwords) { - struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); + struct rvt_qp *qp = packet->qp; + struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd); struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); struct ib_header *hdr = &opa_hdr->ibh; struct ib_other_headers *ohdr; @@ -804,19 +799,20 @@ static inline void hfi1_make_rc_ack_9B(struct rvt_qp *qp, hfi1_make_bth_aeth(qp, ohdr, bth0, bth1); } -static inline void hfi1_make_rc_ack_16B(struct rvt_qp *qp, +static inline void hfi1_make_rc_ack_16B(struct hfi1_packet *packet, struct hfi1_opa_header *opa_hdr, u8 sc5, bool is_fecn, u64 *pbc_flags, u32 *hwords, u32 *nwords) { - struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); + struct rvt_qp *qp = packet->qp; + struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd); struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); struct hfi1_16b_header *hdr = &opa_hdr->opah; struct ib_other_headers *ohdr; u32 bth0, bth1 = 0; u16 len, pkey; - u8 becn = !!is_fecn; + bool becn = is_fecn; u8 l4 = OPA_16B_L4_IB_LOCAL; u8 extra_bytes; @@ -843,11 +839,11 @@ static inline void hfi1_make_rc_ack_16B(struct rvt_qp *qp, /* Convert dwords to flits */ len = (*hwords + *nwords) >> 1; - hfi1_make_16b_hdr(hdr, - ppd->lid | rdma_ah_get_path_bits(&qp->remote_ah_attr), + hfi1_make_16b_hdr(hdr, ppd->lid | + (rdma_ah_get_path_bits(&qp->remote_ah_attr) & + ((1 << ppd->lmc) - 1)), opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr), - 16B), - len, pkey, becn, 0, l4, sc5); + 16B), len, pkey, becn, 0, l4, sc5); bth0 = pkey | (OP(ACKNOWLEDGE) << 24); bth0 |= extra_bytes << 20; @@ -856,7 +852,7 @@ static inline void hfi1_make_rc_ack_16B(struct rvt_qp *qp, hfi1_make_bth_aeth(qp, ohdr, bth0, bth1); } -typedef void (*hfi1_make_rc_ack)(struct rvt_qp *qp, +typedef void (*hfi1_make_rc_ack)(struct hfi1_packet *packet, struct hfi1_opa_header *opa_hdr, u8 sc5, bool is_fecn, u64 *pbc_flags, u32 *hwords, @@ -876,9 +872,10 @@ static const hfi1_make_rc_ack hfi1_make_rc_ack_tbl[2] = { * Note that RDMA reads and atomics are handled in the * send side QP state and send engine. */ -void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd, - struct rvt_qp *qp, bool is_fecn) +void hfi1_send_rc_ack(struct hfi1_packet *packet, bool is_fecn) { + struct hfi1_ctxtdata *rcd = packet->rcd; + struct rvt_qp *qp = packet->qp; struct hfi1_ibport *ibp = rcd_to_iport(rcd); struct hfi1_qp_priv *priv = qp->priv; struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); @@ -895,14 +892,13 @@ void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd, /* Don't send ACK or NAK if a RDMA read or atomic is pending. */ if (qp->s_flags & RVT_S_RESP_PENDING) { - hfi1_queue_rc_ack(qp, is_fecn); + hfi1_queue_rc_ack(packet, is_fecn); return; } /* Ensure s_rdma_ack_cnt changes are committed */ - smp_read_barrier_depends(); if (qp->s_rdma_ack_cnt) { - hfi1_queue_rc_ack(qp, is_fecn); + hfi1_queue_rc_ack(packet, is_fecn); return; } @@ -911,7 +907,7 @@ void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd, return; /* Make the appropriate header */ - hfi1_make_rc_ack_tbl[priv->hdr_type](qp, &opa_hdr, sc5, is_fecn, + hfi1_make_rc_ack_tbl[priv->hdr_type](packet, &opa_hdr, sc5, is_fecn, &pbc_flags, &hwords, &nwords); plen = 2 /* PBC */ + hwords + nwords; @@ -925,7 +921,7 @@ void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd, * so that when enough buffer space becomes available, * the ACK is sent ahead of other outgoing packets. */ - hfi1_queue_rc_ack(qp, is_fecn); + hfi1_queue_rc_ack(packet, is_fecn); return; } trace_ack_output_ibhdr(dd_from_ibdev(qp->ibqp.device), @@ -1543,7 +1539,7 @@ static void rc_rcv_resp(struct hfi1_packet *packet) void *data = packet->payload; u32 tlen = packet->tlen; struct rvt_qp *qp = packet->qp; - struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); + struct hfi1_ibport *ibp; struct ib_other_headers *ohdr = packet->ohdr; struct rvt_swqe *wqe; enum ib_wc_status status; @@ -1562,7 +1558,6 @@ static void rc_rcv_resp(struct hfi1_packet *packet) trace_hfi1_ack(qp, psn); /* Ignore invalid responses. */ - smp_read_barrier_depends(); /* see post_one_send */ if (cmp_psn(psn, READ_ONCE(qp->s_next_psn)) >= 0) goto ack_done; @@ -1701,6 +1696,7 @@ ack_op_err: goto ack_err; ack_seq_err: + ibp = rcd_to_iport(rcd); rdma_seq_err(qp, ibp, psn, rcd); goto ack_done; @@ -2041,7 +2037,6 @@ void hfi1_rc_rcv(struct hfi1_packet *packet) struct rvt_qp *qp = packet->qp; struct hfi1_ibport *ibp = rcd_to_iport(rcd); struct ib_other_headers *ohdr = packet->ohdr; - u32 bth0 = be32_to_cpu(ohdr->bth[0]); u32 opcode = packet->opcode; u32 hdrsize = packet->hlen; u32 psn = ib_bth_get_psn(packet->ohdr); @@ -2239,7 +2234,7 @@ send_last: wc.port_num = 0; /* Signal completion event if the solicited bit is set. */ rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, - (bth0 & IB_BTH_SOLICITED) != 0); + ib_bth_is_solicited(ohdr)); break; case OP(RDMA_WRITE_ONLY): @@ -2483,7 +2478,7 @@ nack_acc: qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR; qp->r_ack_psn = qp->r_psn; send_ack: - hfi1_send_rc_ack(rcd, qp, is_fecn); + hfi1_send_rc_ack(packet, is_fecn); } void hfi1_rc_hdrerr( diff --git a/drivers/infiniband/hw/hfi1/ruc.c b/drivers/infiniband/hw/hfi1/ruc.c index 2c7fc6e331ea..3daa94bdae3a 100644 --- a/drivers/infiniband/hw/hfi1/ruc.c +++ b/drivers/infiniband/hw/hfi1/ruc.c @@ -225,19 +225,8 @@ int hfi1_ruc_check_hdr(struct hfi1_ibport *ibp, struct hfi1_packet *packet) u32 dlid = packet->dlid; u32 slid = packet->slid; u32 sl = packet->sl; - int migrated; - u32 bth0, bth1; - u16 pkey; - - bth0 = be32_to_cpu(packet->ohdr->bth[0]); - bth1 = be32_to_cpu(packet->ohdr->bth[1]); - if (packet->etype == RHF_RCV_TYPE_BYPASS) { - pkey = hfi1_16B_get_pkey(packet->hdr); - migrated = bth1 & OPA_BTH_MIG_REQ; - } else { - pkey = ib_bth_get_pkey(packet->ohdr); - migrated = bth0 & IB_BTH_MIG_REQ; - } + bool migrated = packet->migrated; + u16 pkey = packet->pkey; if (qp->s_mig_state == IB_MIG_ARMED && migrated) { if (!packet->grh) { @@ -362,7 +351,6 @@ static void ruc_loopback(struct rvt_qp *sqp) sqp->s_flags |= RVT_S_BUSY; again: - smp_read_barrier_depends(); /* see post_one_send() */ if (sqp->s_last == READ_ONCE(sqp->s_head)) goto clr_busy; wqe = rvt_get_swqe_ptr(sqp, sqp->s_last); @@ -757,19 +745,18 @@ static inline void hfi1_make_ruc_header_16B(struct rvt_qp *qp, u32 slid; u16 pkey = hfi1_get_pkey(ibp, qp->s_pkey_index); u8 l4 = OPA_16B_L4_IB_LOCAL; - u8 extra_bytes = hfi1_get_16b_padding((qp->s_hdrwords << 2), - ps->s_txreq->s_cur_size); + u8 extra_bytes = hfi1_get_16b_padding( + (ps->s_txreq->hdr_dwords << 2), + ps->s_txreq->s_cur_size); u32 nwords = SIZE_OF_CRC + ((ps->s_txreq->s_cur_size + extra_bytes + SIZE_OF_LT) >> 2); - u8 becn = 0; + bool becn = false; if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) && hfi1_check_mcast(rdma_ah_get_dlid(&qp->remote_ah_attr))) { struct ib_grh *grh; struct ib_global_route *grd = rdma_ah_retrieve_grh(&qp->remote_ah_attr); - int hdrwords; - /* * Ensure OPA GIDs are transformed to IB gids * before creating the GRH. @@ -778,9 +765,10 @@ static inline void hfi1_make_ruc_header_16B(struct rvt_qp *qp, grd->sgid_index = 0; grh = &ps->s_txreq->phdr.hdr.opah.u.l.grh; l4 = OPA_16B_L4_IB_GLOBAL; - hdrwords = qp->s_hdrwords - 4; - qp->s_hdrwords += hfi1_make_grh(ibp, grh, grd, - hdrwords, nwords); + ps->s_txreq->hdr_dwords += + hfi1_make_grh(ibp, grh, grd, + ps->s_txreq->hdr_dwords - LRH_16B_DWORDS, + nwords); middle = 0; } @@ -799,7 +787,7 @@ static inline void hfi1_make_ruc_header_16B(struct rvt_qp *qp, if (qp->s_flags & RVT_S_ECN) { qp->s_flags &= ~RVT_S_ECN; /* we recently received a FECN, so return a BECN */ - becn = 1; + becn = true; } hfi1_make_ruc_bth(qp, ohdr, bth0, bth1, bth2); @@ -814,7 +802,7 @@ static inline void hfi1_make_ruc_header_16B(struct rvt_qp *qp, slid, opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr), 16B), - (qp->s_hdrwords + nwords) >> 1, + (ps->s_txreq->hdr_dwords + nwords) >> 1, pkey, becn, 0, l4, priv->s_sc); } @@ -834,13 +822,13 @@ static inline void hfi1_make_ruc_header_9B(struct rvt_qp *qp, if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)) { struct ib_grh *grh = &ps->s_txreq->phdr.hdr.ibh.u.l.grh; - int hdrwords = qp->s_hdrwords - 2; lrh0 = HFI1_LRH_GRH; - qp->s_hdrwords += + ps->s_txreq->hdr_dwords += hfi1_make_grh(ibp, grh, rdma_ah_read_grh(&qp->remote_ah_attr), - hdrwords, nwords); + ps->s_txreq->hdr_dwords - LRH_9B_DWORDS, + nwords); middle = 0; } lrh0 |= (priv->s_sc & 0xf) << 12 | @@ -866,7 +854,7 @@ static inline void hfi1_make_ruc_header_9B(struct rvt_qp *qp, hfi1_make_ruc_bth(qp, ohdr, bth0, bth1, bth2); hfi1_make_ib_hdr(&ps->s_txreq->phdr.hdr.ibh, lrh0, - qp->s_hdrwords + nwords, + ps->s_txreq->hdr_dwords + nwords, opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr), 9B), ppd_from_ibp(ibp)->lid | rdma_ah_get_path_bits(&qp->remote_ah_attr)); @@ -1031,7 +1019,7 @@ void hfi1_do_send(struct rvt_qp *qp, bool in_thread) ps.s_txreq = get_waiting_verbs_txreq(qp); do { /* Check for a constructed packet to be sent. */ - if (qp->s_hdrwords != 0) { + if (ps.s_txreq) { spin_unlock_irqrestore(&qp->s_lock, ps.flags); /* * If the packet cannot be sent now, return and @@ -1039,8 +1027,6 @@ void hfi1_do_send(struct rvt_qp *qp, bool in_thread) */ if (hfi1_verbs_send(qp, &ps)) return; - /* Record that s_ahg is empty. */ - qp->s_hdrwords = 0; /* allow other tasks to run */ if (schedule_send_yield(qp, &ps)) return; diff --git a/drivers/infiniband/hw/hfi1/sdma.c b/drivers/infiniband/hw/hfi1/sdma.c index 31c8f89b5fc8..1f203309cf24 100644 --- a/drivers/infiniband/hw/hfi1/sdma.c +++ b/drivers/infiniband/hw/hfi1/sdma.c @@ -553,7 +553,6 @@ static void sdma_hw_clean_up_task(unsigned long opaque) static inline struct sdma_txreq *get_txhead(struct sdma_engine *sde) { - smp_read_barrier_depends(); /* see sdma_update_tail() */ return sde->tx_ring[sde->tx_head & sde->sdma_mask]; } @@ -1276,13 +1275,15 @@ bail: return -ENOMEM; } -/* - * Clean up allocated memory. - * - * This routine is can be called regardless of the success of sdma_init() +/** + * sdma_clean() Clean up allocated memory + * @dd: struct hfi1_devdata + * @num_engines: num sdma engines * + * This routine can be called regardless of the success of + * sdma_init() */ -static void sdma_clean(struct hfi1_devdata *dd, size_t num_engines) +void sdma_clean(struct hfi1_devdata *dd, size_t num_engines) { size_t i; struct sdma_engine *sde; @@ -1387,7 +1388,8 @@ int sdma_init(struct hfi1_devdata *dd, u8 port) num_engines, descq_cnt); /* alloc memory for array of send engines */ - dd->per_sdma = kcalloc(num_engines, sizeof(*dd->per_sdma), GFP_KERNEL); + dd->per_sdma = kcalloc_node(num_engines, sizeof(*dd->per_sdma), + GFP_KERNEL, dd->node); if (!dd->per_sdma) return ret; @@ -1618,7 +1620,6 @@ void sdma_exit(struct hfi1_devdata *dd) */ sdma_finalput(&sde->state); } - sdma_clean(dd, dd->num_sdma); } /* diff --git a/drivers/infiniband/hw/hfi1/sdma.h b/drivers/infiniband/hw/hfi1/sdma.h index 374c59784950..46c775f255d1 100644 --- a/drivers/infiniband/hw/hfi1/sdma.h +++ b/drivers/infiniband/hw/hfi1/sdma.h @@ -420,6 +420,7 @@ struct sdma_engine { int sdma_init(struct hfi1_devdata *dd, u8 port); void sdma_start(struct hfi1_devdata *dd); void sdma_exit(struct hfi1_devdata *dd); +void sdma_clean(struct hfi1_devdata *dd, size_t num_engines); void sdma_all_running(struct hfi1_devdata *dd); void sdma_all_idle(struct hfi1_devdata *dd); void sdma_freeze_notify(struct hfi1_devdata *dd, int go_idle); diff --git a/drivers/infiniband/hw/hfi1/trace.c b/drivers/infiniband/hw/hfi1/trace.c index 959a80429ee9..89bd9851065b 100644 --- a/drivers/infiniband/hw/hfi1/trace.c +++ b/drivers/infiniband/hw/hfi1/trace.c @@ -138,7 +138,7 @@ static const char *parse_syndrome(u8 syndrome) } void hfi1_trace_parse_9b_bth(struct ib_other_headers *ohdr, - u8 *ack, u8 *becn, u8 *fecn, u8 *mig, + u8 *ack, bool *becn, bool *fecn, u8 *mig, u8 *se, u8 *pad, u8 *opcode, u8 *tver, u16 *pkey, u32 *psn, u32 *qpn) { @@ -184,7 +184,7 @@ void hfi1_trace_parse_9b_hdr(struct ib_header *hdr, bool sc5, } void hfi1_trace_parse_16b_hdr(struct hfi1_16b_header *hdr, - u8 *age, u8 *becn, u8 *fecn, + u8 *age, bool *becn, bool *fecn, u8 *l4, u8 *rc, u8 *sc, u16 *entropy, u16 *len, u16 *pkey, u32 *dlid, u32 *slid) @@ -207,7 +207,7 @@ void hfi1_trace_parse_16b_hdr(struct hfi1_16b_header *hdr, #define LRH_16B_PRN "age:%d becn:%d fecn:%d l4:%d " \ "rc:%d sc:%d pkey:0x%.4x entropy:0x%.4x" const char *hfi1_trace_fmt_lrh(struct trace_seq *p, bool bypass, - u8 age, u8 becn, u8 fecn, u8 l4, + u8 age, bool becn, bool fecn, u8 l4, u8 lnh, const char *lnh_name, u8 lver, u8 rc, u8 sc, u8 sl, u16 entropy, u16 len, u16 pkey, u32 dlid, u32 slid) @@ -235,7 +235,7 @@ const char *hfi1_trace_fmt_lrh(struct trace_seq *p, bool bypass, "op:0x%.2x,%s se:%d m:%d pad:%d tver:%d " \ "qpn:0x%.6x a:%d psn:0x%.8x" const char *hfi1_trace_fmt_bth(struct trace_seq *p, bool bypass, - u8 ack, u8 becn, u8 fecn, u8 mig, + u8 ack, bool becn, bool fecn, u8 mig, u8 se, u8 pad, u8 opcode, const char *opname, u8 tver, u16 pkey, u32 psn, u32 qpn) { diff --git a/drivers/infiniband/hw/hfi1/trace_ibhdrs.h b/drivers/infiniband/hw/hfi1/trace_ibhdrs.h index fb631278eccd..2847626d3819 100644 --- a/drivers/infiniband/hw/hfi1/trace_ibhdrs.h +++ b/drivers/infiniband/hw/hfi1/trace_ibhdrs.h @@ -101,7 +101,7 @@ u8 hfi1_trace_opa_hdr_len(struct hfi1_opa_header *opah); u8 hfi1_trace_packet_hdr_len(struct hfi1_packet *packet); const char *hfi1_trace_get_packet_l4_str(u8 l4); void hfi1_trace_parse_9b_bth(struct ib_other_headers *ohdr, - u8 *ack, u8 *becn, u8 *fecn, u8 *mig, + u8 *ack, bool *becn, bool *fecn, u8 *mig, u8 *se, u8 *pad, u8 *opcode, u8 *tver, u16 *pkey, u32 *psn, u32 *qpn); void hfi1_trace_parse_9b_hdr(struct ib_header *hdr, bool sc5, @@ -112,19 +112,19 @@ void hfi1_trace_parse_16b_bth(struct ib_other_headers *ohdr, u8 *pad, u8 *se, u8 *tver, u32 *psn, u32 *qpn); void hfi1_trace_parse_16b_hdr(struct hfi1_16b_header *hdr, - u8 *age, u8 *becn, u8 *fecn, + u8 *age, bool *becn, bool *fecn, u8 *l4, u8 *rc, u8 *sc, u16 *entropy, u16 *len, u16 *pkey, u32 *dlid, u32 *slid); const char *hfi1_trace_fmt_lrh(struct trace_seq *p, bool bypass, - u8 age, u8 becn, u8 fecn, u8 l4, + u8 age, bool becn, bool fecn, u8 l4, u8 lnh, const char *lnh_name, u8 lver, u8 rc, u8 sc, u8 sl, u16 entropy, u16 len, u16 pkey, u32 dlid, u32 slid); const char *hfi1_trace_fmt_bth(struct trace_seq *p, bool bypass, - u8 ack, u8 becn, u8 fecn, u8 mig, + u8 ack, bool becn, bool fecn, u8 mig, u8 se, u8 pad, u8 opcode, const char *opname, u8 tver, u16 pkey, u32 psn, u32 qpn); @@ -148,8 +148,8 @@ DECLARE_EVENT_CLASS(hfi1_input_ibhdr_template, __field(u8, etype) __field(u8, ack) __field(u8, age) - __field(u8, becn) - __field(u8, fecn) + __field(bool, becn) + __field(bool, fecn) __field(u8, l2) __field(u8, l4) __field(u8, lnh) @@ -290,8 +290,8 @@ DECLARE_EVENT_CLASS(hfi1_output_ibhdr_template, __field(u8, hdr_type) __field(u8, ack) __field(u8, age) - __field(u8, becn) - __field(u8, fecn) + __field(bool, becn) + __field(bool, fecn) __field(u8, l4) __field(u8, lnh) __field(u8, lver) diff --git a/drivers/infiniband/hw/hfi1/trace_rx.h b/drivers/infiniband/hw/hfi1/trace_rx.h index 4d487fee105d..7eceb57e0415 100644 --- a/drivers/infiniband/hw/hfi1/trace_rx.h +++ b/drivers/infiniband/hw/hfi1/trace_rx.h @@ -63,17 +63,9 @@ __print_symbolic(type, \ #define TRACE_SYSTEM hfi1_rx TRACE_EVENT(hfi1_rcvhdr, - TP_PROTO(struct hfi1_devdata *dd, - u32 ctxt, - u64 eflags, - u32 etype, - u32 hlen, - u32 tlen, - u32 updegr, - u32 etail - ), - TP_ARGS(dd, ctxt, eflags, etype, hlen, tlen, updegr, etail), - TP_STRUCT__entry(DD_DEV_ENTRY(dd) + TP_PROTO(struct hfi1_packet *packet), + TP_ARGS(packet), + TP_STRUCT__entry(DD_DEV_ENTRY(packet->rcd->dd) __field(u64, eflags) __field(u32, ctxt) __field(u32, etype) @@ -82,14 +74,14 @@ TRACE_EVENT(hfi1_rcvhdr, __field(u32, updegr) __field(u32, etail) ), - TP_fast_assign(DD_DEV_ASSIGN(dd); - __entry->eflags = eflags; - __entry->ctxt = ctxt; - __entry->etype = etype; - __entry->hlen = hlen; - __entry->tlen = tlen; - __entry->updegr = updegr; - __entry->etail = etail; + TP_fast_assign(DD_DEV_ASSIGN(packet->rcd->dd); + __entry->eflags = rhf_err_flags(packet->rhf); + __entry->ctxt = packet->rcd->ctxt; + __entry->etype = packet->etype; + __entry->hlen = packet->hlen; + __entry->tlen = packet->tlen; + __entry->updegr = packet->updegr; + __entry->etail = rhf_egr_index(packet->rhf); ), TP_printk( "[%s] ctxt %d eflags 0x%llx etype %d,%s hlen %d tlen %d updegr %d etail %d", diff --git a/drivers/infiniband/hw/hfi1/uc.c b/drivers/infiniband/hw/hfi1/uc.c index 991bbee04821..9d7a3110c14c 100644 --- a/drivers/infiniband/hw/hfi1/uc.c +++ b/drivers/infiniband/hw/hfi1/uc.c @@ -79,7 +79,6 @@ int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND)) goto bail; /* We are in the error state, flush the work request. */ - smp_read_barrier_depends(); /* see post_one_send() */ if (qp->s_last == READ_ONCE(qp->s_head)) goto bail; /* If DMAs are in progress, we can't flush immediately. */ @@ -119,7 +118,6 @@ int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) RVT_PROCESS_NEXT_SEND_OK)) goto bail; /* Check if send work queue is empty. */ - smp_read_barrier_depends(); /* see post_one_send() */ if (qp->s_cur == READ_ONCE(qp->s_head)) { clear_ahg(qp); goto bail; @@ -146,7 +144,6 @@ int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) : IB_WC_SUCCESS); if (local_ops) atomic_dec(&qp->local_ops_pending); - qp->s_hdrwords = 0; goto done_free_tx; } /* @@ -269,14 +266,12 @@ int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) break; } qp->s_len -= len; - qp->s_hdrwords = hwords; + ps->s_txreq->hdr_dwords = hwords; ps->s_txreq->sde = priv->s_sde; ps->s_txreq->ss = &qp->s_sge; ps->s_txreq->s_cur_size = len; hfi1_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24), mask_psn(qp->s_psn++), middle, ps); - /* pbc */ - ps->s_txreq->hdr_dwords = qp->s_hdrwords + 2; return 1; done_free_tx: @@ -290,7 +285,6 @@ bail: bail_no_tx: ps->s_txreq = NULL; qp->s_flags &= ~RVT_S_BUSY; - qp->s_hdrwords = 0; return 0; } @@ -482,8 +476,7 @@ last_imm: wc.port_num = 0; /* Signal completion event if the solicited bit is set. */ rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, - (ohdr->bth[0] & - cpu_to_be32(IB_BTH_SOLICITED)) != 0); + ib_bth_is_solicited(ohdr)); break; case OP(RDMA_WRITE_FIRST): diff --git a/drivers/infiniband/hw/hfi1/ud.c b/drivers/infiniband/hw/hfi1/ud.c index beb5091eccca..bcf3b0bebac8 100644 --- a/drivers/infiniband/hw/hfi1/ud.c +++ b/drivers/infiniband/hw/hfi1/ud.c @@ -340,15 +340,16 @@ void hfi1_make_ud_req_9B(struct rvt_qp *qp, struct hfi1_pkt_state *ps, extra_bytes = -wqe->length & 3; nwords = ((wqe->length + extra_bytes) >> 2) + SIZE_OF_CRC; /* header size in dwords LRH+BTH+DETH = (8+12+8)/4. */ - qp->s_hdrwords = 7; + ps->s_txreq->hdr_dwords = 7; if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) - qp->s_hdrwords++; + ps->s_txreq->hdr_dwords++; if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) { grh = &ps->s_txreq->phdr.hdr.ibh.u.l.grh; - qp->s_hdrwords += hfi1_make_grh(ibp, grh, - rdma_ah_read_grh(ah_attr), - qp->s_hdrwords - 2, nwords); + ps->s_txreq->hdr_dwords += + hfi1_make_grh(ibp, grh, rdma_ah_read_grh(ah_attr), + ps->s_txreq->hdr_dwords - LRH_9B_DWORDS, + nwords); lrh0 = HFI1_LRH_GRH; ohdr = &ps->s_txreq->phdr.hdr.ibh.u.l.oth; } else { @@ -381,7 +382,7 @@ void hfi1_make_ud_req_9B(struct rvt_qp *qp, struct hfi1_pkt_state *ps, } } hfi1_make_bth_deth(qp, wqe, ohdr, &pkey, extra_bytes, false); - len = qp->s_hdrwords + nwords; + len = ps->s_txreq->hdr_dwords + nwords; /* Setup the packet */ ps->s_txreq->phdr.hdr.hdr_type = HFI1_PKT_TYPE_9B; @@ -405,12 +406,12 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps, ppd = ppd_from_ibp(ibp); ah_attr = &ibah_to_rvtah(wqe->ud_wr.ah)->attr; /* header size in dwords 16B LRH+BTH+DETH = (16+12+8)/4. */ - qp->s_hdrwords = 9; + ps->s_txreq->hdr_dwords = 9; if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) - qp->s_hdrwords++; + ps->s_txreq->hdr_dwords++; /* SW provides space for CRC and LT for bypass packets. */ - extra_bytes = hfi1_get_16b_padding((qp->s_hdrwords << 2), + extra_bytes = hfi1_get_16b_padding((ps->s_txreq->hdr_dwords << 2), wqe->length); nwords = ((wqe->length + extra_bytes + SIZE_OF_LT) >> 2) + SIZE_OF_CRC; @@ -428,8 +429,10 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps, grd->sgid_index = 0; } grh = &ps->s_txreq->phdr.hdr.opah.u.l.grh; - qp->s_hdrwords += hfi1_make_grh(ibp, grh, grd, - qp->s_hdrwords - 4, nwords); + ps->s_txreq->hdr_dwords += hfi1_make_grh( + ibp, grh, grd, + ps->s_txreq->hdr_dwords - LRH_16B_DWORDS, + nwords); ohdr = &ps->s_txreq->phdr.hdr.opah.u.l.oth; l4 = OPA_16B_L4_IB_GLOBAL; } else { @@ -452,7 +455,7 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps, hfi1_make_bth_deth(qp, wqe, ohdr, &pkey, extra_bytes, true); /* Convert dwords to flits */ - len = (qp->s_hdrwords + nwords) >> 1; + len = (ps->s_txreq->hdr_dwords + nwords) >> 1; /* Setup the packet */ ps->s_txreq->phdr.hdr.hdr_type = HFI1_PKT_TYPE_16B; @@ -486,7 +489,6 @@ int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND)) goto bail; /* We are in the error state, flush the work request. */ - smp_read_barrier_depends(); /* see post_one_send */ if (qp->s_last == READ_ONCE(qp->s_head)) goto bail; /* If DMAs are in progress, we can't flush immediately. */ @@ -500,7 +502,6 @@ int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) } /* see post_one_send() */ - smp_read_barrier_depends(); if (qp->s_cur == READ_ONCE(qp->s_head)) goto bail; @@ -564,8 +565,6 @@ int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) priv->s_ahg->ahgcount = 0; priv->s_ahg->ahgidx = 0; priv->s_ahg->tx_flags = 0; - /* pbc */ - ps->s_txreq->hdr_dwords = qp->s_hdrwords + 2; return 1; @@ -580,7 +579,6 @@ bail: bail_no_tx: ps->s_txreq = NULL; qp->s_flags &= ~RVT_S_BUSY; - qp->s_hdrwords = 0; return 0; } @@ -651,7 +649,8 @@ void return_cnp_16B(struct hfi1_ibport *ibp, struct rvt_qp *qp, struct ib_grh *grh = &hdr.u.l.grh; grh->version_tclass_flow = old_grh->version_tclass_flow; - grh->paylen = cpu_to_be16((hwords - 4 + nwords) << 2); + grh->paylen = cpu_to_be16( + (hwords - LRH_16B_DWORDS + nwords) << 2); grh->hop_limit = 0xff; grh->sgid = old_grh->dgid; grh->dgid = old_grh->sgid; @@ -705,7 +704,8 @@ void return_cnp(struct hfi1_ibport *ibp, struct rvt_qp *qp, u32 remote_qpn, struct ib_grh *grh = &hdr.u.l.grh; grh->version_tclass_flow = old_grh->version_tclass_flow; - grh->paylen = cpu_to_be16((hwords - 2 + SIZE_OF_CRC) << 2); + grh->paylen = cpu_to_be16( + (hwords - LRH_9B_DWORDS + SIZE_OF_CRC) << 2); grh->hop_limit = 0xff; grh->sgid = old_grh->dgid; grh->dgid = old_grh->sgid; @@ -1048,8 +1048,7 @@ void hfi1_ud_rcv(struct hfi1_packet *packet) wc.port_num = qp->port_num; /* Signal completion event if the solicited bit is set. */ rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, - (ohdr->bth[0] & - cpu_to_be32(IB_BTH_SOLICITED)) != 0); + ib_bth_is_solicited(ohdr)); return; drop: diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c index a38785e224cc..471d55c50066 100644 --- a/drivers/infiniband/hw/hfi1/verbs.c +++ b/drivers/infiniband/hw/hfi1/verbs.c @@ -835,7 +835,7 @@ static int build_verbs_tx_desc( { int ret = 0; struct hfi1_sdma_header *phdr = &tx->phdr; - u16 hdrbytes = tx->hdr_dwords << 2; + u16 hdrbytes = (tx->hdr_dwords + sizeof(pbc) / 4) << 2; u8 extra_bytes = 0; if (tx->phdr.hdr.hdr_type) { @@ -901,7 +901,7 @@ int hfi1_verbs_send_dma(struct rvt_qp *qp, struct hfi1_pkt_state *ps, { struct hfi1_qp_priv *priv = qp->priv; struct hfi1_ahg_info *ahg_info = priv->s_ahg; - u32 hdrwords = qp->s_hdrwords; + u32 hdrwords = ps->s_txreq->hdr_dwords; u32 len = ps->s_txreq->s_cur_size; u32 plen; struct hfi1_ibdev *dev = ps->dev; @@ -919,7 +919,7 @@ int hfi1_verbs_send_dma(struct rvt_qp *qp, struct hfi1_pkt_state *ps, } else { dwords = (len + 3) >> 2; } - plen = hdrwords + dwords + 2; + plen = hdrwords + dwords + sizeof(pbc) / 4; tx = ps->s_txreq; if (!sdma_txreq_built(&tx->txreq)) { @@ -1038,7 +1038,7 @@ int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps, u64 pbc) { struct hfi1_qp_priv *priv = qp->priv; - u32 hdrwords = qp->s_hdrwords; + u32 hdrwords = ps->s_txreq->hdr_dwords; struct rvt_sge_state *ss = ps->s_txreq->ss; u32 len = ps->s_txreq->s_cur_size; u32 dwords; @@ -1064,7 +1064,7 @@ int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps, dwords = (len + 3) >> 2; hdr = (u32 *)&ps->s_txreq->phdr.hdr.ibh; } - plen = hdrwords + dwords + 2; + plen = hdrwords + dwords + sizeof(pbc) / 4; /* only RC/UC use complete */ switch (qp->ibqp.qp_type) { @@ -1486,7 +1486,7 @@ static int query_port(struct rvt_dev_info *rdi, u8 port_num, props->max_mtu = mtu_to_enum((!valid_ib_mtu(hfi1_max_mtu) ? 4096 : hfi1_max_mtu), IB_MTU_4096); props->active_mtu = !valid_ib_mtu(ppd->ibmtu) ? props->max_mtu : - mtu_to_enum(ppd->ibmtu, IB_MTU_2048); + mtu_to_enum(ppd->ibmtu, IB_MTU_4096); /* * sm_lid of 0xFFFF needs special handling so that it can @@ -1844,7 +1844,6 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd) struct hfi1_ibport *ibp = &ppd->ibport_data; unsigned i; int ret; - size_t lcpysz = IB_DEVICE_NAME_MAX; for (i = 0; i < dd->num_pports; i++) init_ibport(ppd + i); @@ -1872,8 +1871,6 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd) */ if (!ib_hfi1_sys_image_guid) ib_hfi1_sys_image_guid = ibdev->node_guid; - lcpysz = strlcpy(ibdev->name, class_name(), lcpysz); - strlcpy(ibdev->name + lcpysz, "_%d", IB_DEVICE_NAME_MAX - lcpysz); ibdev->owner = THIS_MODULE; ibdev->phys_port_cnt = dd->num_pports; ibdev->dev.parent = &dd->pcidev->dev; @@ -1893,7 +1890,6 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd) * Fill in rvt info object. */ dd->verbs_dev.rdi.driver_f.port_callback = hfi1_create_port_files; - dd->verbs_dev.rdi.driver_f.get_card_name = get_card_name; dd->verbs_dev.rdi.driver_f.get_pci_dev = get_pci_dev; dd->verbs_dev.rdi.driver_f.check_ah = hfi1_check_ah; dd->verbs_dev.rdi.driver_f.notify_new_ah = hfi1_notify_new_ah; diff --git a/drivers/infiniband/hw/hfi1/verbs.h b/drivers/infiniband/hw/hfi1/verbs.h index 87d1285a3340..2d787b8346ca 100644 --- a/drivers/infiniband/hw/hfi1/verbs.h +++ b/drivers/infiniband/hw/hfi1/verbs.h @@ -105,6 +105,11 @@ enum { HFI1_HAS_GRH = (1 << 0), }; +#define LRH_16B_BYTES (FIELD_SIZEOF(struct hfi1_16b_header, lrh)) +#define LRH_16B_DWORDS (LRH_16B_BYTES / sizeof(u32)) +#define LRH_9B_BYTES (FIELD_SIZEOF(struct ib_header, lrh)) +#define LRH_9B_DWORDS (LRH_9B_BYTES / sizeof(u32)) + struct hfi1_16b_header { u32 lrh[4]; union { @@ -246,17 +251,6 @@ static inline struct rvt_qp *iowait_to_qp(struct iowait *s_iowait) } /* - * Send if not busy or waiting for I/O and either - * a RC response is pending or we can process send work requests. - */ -static inline int hfi1_send_ok(struct rvt_qp *qp) -{ - return !(qp->s_flags & (RVT_S_BUSY | RVT_S_ANY_WAIT_IO)) && - (qp->s_hdrwords || (qp->s_flags & RVT_S_RESP_PENDING) || - !(qp->s_flags & RVT_S_ANY_WAIT_SEND)); -} - -/* * This must be called with s_lock held. */ void hfi1_bad_pkey(struct hfi1_ibport *ibp, u32 key, u32 sl, @@ -369,8 +363,7 @@ void hfi1_do_send(struct rvt_qp *qp, bool in_thread); void hfi1_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe, enum ib_wc_status status); -void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp, - bool is_fecn); +void hfi1_send_rc_ack(struct hfi1_packet *packet, bool is_fecn); int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps); @@ -416,6 +409,11 @@ static inline void cacheless_memcpy(void *dst, void *src, size_t n) __copy_user_nocache(dst, (void __user *)src, n, 0); } +static inline bool opa_bth_is_migration(struct ib_other_headers *ohdr) +{ + return ohdr->bth[1] & cpu_to_be32(OPA_BTH_MIG_REQ); +} + extern const enum ib_wc_opcode ib_hfi1_wc_opcode[]; extern const u8 hdr_len_by_opcode[]; diff --git a/drivers/infiniband/hw/hfi1/verbs_txreq.h b/drivers/infiniband/hw/hfi1/verbs_txreq.h index cec7a4b34d16..729244c3086c 100644 --- a/drivers/infiniband/hw/hfi1/verbs_txreq.h +++ b/drivers/infiniband/hw/hfi1/verbs_txreq.h @@ -113,6 +113,13 @@ static inline struct verbs_txreq *get_waiting_verbs_txreq(struct rvt_qp *qp) return NULL; } +static inline bool verbs_txreq_queued(struct rvt_qp *qp) +{ + struct hfi1_qp_priv *priv = qp->priv; + + return iowait_packet_queued(&priv->s_iowait); +} + void hfi1_put_txreq(struct verbs_txreq *tx); int verbs_txreq_init(struct hfi1_ibdev *dev); void verbs_txreq_exit(struct hfi1_ibdev *dev); |