diff options
Diffstat (limited to 'drivers/net/ethernet/pensando/ionic')
| -rw-r--r-- | drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c | 4 | ||||
| -rw-r--r-- | drivers/net/ethernet/pensando/ionic/ionic_dev.c | 12 | ||||
| -rw-r--r-- | drivers/net/ethernet/pensando/ionic/ionic_dev.h | 23 | ||||
| -rw-r--r-- | drivers/net/ethernet/pensando/ionic/ionic_ethtool.c | 16 | ||||
| -rw-r--r-- | drivers/net/ethernet/pensando/ionic/ionic_if.h | 28 | ||||
| -rw-r--r-- | drivers/net/ethernet/pensando/ionic/ionic_lif.c | 83 | ||||
| -rw-r--r-- | drivers/net/ethernet/pensando/ionic/ionic_lif.h | 23 | ||||
| -rw-r--r-- | drivers/net/ethernet/pensando/ionic/ionic_main.c | 26 | ||||
| -rw-r--r-- | drivers/net/ethernet/pensando/ionic/ionic_stats.c | 145 | ||||
| -rw-r--r-- | drivers/net/ethernet/pensando/ionic/ionic_txrx.c | 752 |
10 files changed, 579 insertions, 533 deletions
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c b/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c index b0d8499d373b..e4a5416adc80 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c @@ -184,6 +184,10 @@ static int ionic_sriov_configure(struct pci_dev *pdev, int num_vfs) struct device *dev = ionic->dev; int ret = 0; + if (ionic->lif && + test_bit(IONIC_LIF_F_FW_RESET, ionic->lif->state)) + return -EBUSY; + if (num_vfs > 0) { ret = pci_enable_sriov(pdev, num_vfs); if (ret) { diff --git a/drivers/net/ethernet/pensando/ionic/ionic_dev.c b/drivers/net/ethernet/pensando/ionic/ionic_dev.c index fb2b5bf179d7..0532f7cf086d 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_dev.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_dev.c @@ -14,18 +14,20 @@ static void ionic_watchdog_cb(struct timer_list *t) { struct ionic *ionic = from_timer(ionic, t, watchdog_timer); + struct ionic_lif *lif = ionic->lif; int hb; mod_timer(&ionic->watchdog_timer, round_jiffies(jiffies + ionic->watchdog_period)); - if (!ionic->lif) + if (!lif) return; hb = ionic_heartbeat_check(ionic); - if (hb >= 0) - ionic_link_status_check_request(ionic->lif, CAN_NOT_SLEEP); + if (hb >= 0 && + !test_bit(IONIC_LIF_F_FW_RESET, lif->state)) + ionic_link_status_check_request(lif, CAN_NOT_SLEEP); } void ionic_init_devinfo(struct ionic *ionic) @@ -585,9 +587,9 @@ void ionic_q_sg_map(struct ionic_queue *q, void *base, dma_addr_t base_pa) void ionic_q_post(struct ionic_queue *q, bool ring_doorbell, ionic_desc_cb cb, void *cb_arg) { - struct device *dev = q->lif->ionic->dev; struct ionic_desc_info *desc_info; struct ionic_lif *lif = q->lif; + struct device *dev = q->dev; desc_info = &q->info[q->head_idx]; desc_info->cb = cb; @@ -629,7 +631,7 @@ void ionic_q_service(struct ionic_queue *q, struct ionic_cq_info *cq_info, /* stop index must be for a descriptor that is not yet completed */ if (unlikely(!ionic_q_is_posted(q, stop_index))) - dev_err(q->lif->ionic->dev, + dev_err(q->dev, "ionic stop is not posted %s stop %u tail %u head %u\n", q->name, stop_index, q->tail_idx, q->head_idx); diff --git a/drivers/net/ethernet/pensando/ionic/ionic_dev.h b/drivers/net/ethernet/pensando/ionic/ionic_dev.h index 690768ff0143..ca7e55455165 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_dev.h +++ b/drivers/net/ethernet/pensando/ionic/ionic_dev.h @@ -170,11 +170,20 @@ typedef void (*ionic_desc_cb)(struct ionic_queue *q, struct ionic_desc_info *desc_info, struct ionic_cq_info *cq_info, void *cb_arg); -struct ionic_page_info { +#define IONIC_PAGE_SIZE PAGE_SIZE +#define IONIC_PAGE_SPLIT_SZ (PAGE_SIZE / 2) +#define IONIC_PAGE_GFP_MASK (GFP_ATOMIC | __GFP_NOWARN |\ + __GFP_COMP | __GFP_MEMALLOC) + +struct ionic_buf_info { struct page *page; dma_addr_t dma_addr; + u32 page_offset; + u32 len; }; +#define IONIC_MAX_FRAGS (1 + IONIC_TX_MAX_SG_ELEMS_V1) + struct ionic_desc_info { union { void *desc; @@ -187,8 +196,9 @@ struct ionic_desc_info { struct ionic_txq_sg_desc *txq_sg_desc; struct ionic_rxq_sg_desc *rxq_sgl_desc; }; - unsigned int npages; - struct ionic_page_info pages[IONIC_RX_MAX_SG_ELEMS + 1]; + unsigned int bytes; + unsigned int nbufs; + struct ionic_buf_info bufs[IONIC_MAX_FRAGS]; ionic_desc_cb cb; void *cb_arg; }; @@ -199,10 +209,12 @@ struct ionic_queue { struct device *dev; struct ionic_lif *lif; struct ionic_desc_info *info; + u64 dbval; u16 head_idx; u16 tail_idx; unsigned int index; unsigned int num_descs; + unsigned int max_sg_elems; u64 dbell_count; u64 stop; u64 wake; @@ -211,7 +223,6 @@ struct ionic_queue { unsigned int type; unsigned int hw_index; unsigned int hw_type; - u64 dbval; union { void *base; struct ionic_txq_desc *txq; @@ -229,7 +240,7 @@ struct ionic_queue { unsigned int sg_desc_size; unsigned int pid; char name[IONIC_QUEUE_NAME_MAX_SZ]; -}; +} ____cacheline_aligned_in_smp; #define IONIC_INTR_INDEX_NOT_ASSIGNED -1 #define IONIC_INTR_NAME_MAX_SZ 32 @@ -256,7 +267,7 @@ struct ionic_cq { u64 compl_count; void *base; dma_addr_t base_pa; -}; +} ____cacheline_aligned_in_smp; struct ionic; diff --git a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c index 0832bedcb3b4..b1e78b452fad 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c @@ -29,11 +29,9 @@ static void ionic_get_stats_strings(struct ionic_lif *lif, u8 *buf) static void ionic_get_stats(struct net_device *netdev, struct ethtool_stats *stats, u64 *buf) { - struct ionic_lif *lif; + struct ionic_lif *lif = netdev_priv(netdev); u32 i; - lif = netdev_priv(netdev); - memset(buf, 0, stats->n_stats * sizeof(*buf)); for (i = 0; i < ionic_num_stats_grps; i++) ionic_stats_groups[i].get_values(lif, &buf); @@ -209,6 +207,14 @@ static int ionic_get_link_ksettings(struct net_device *netdev, ethtool_link_ksettings_add_link_mode(ks, supported, 10000baseER_Full); break; + case IONIC_XCVR_PID_SFP_10GBASE_T: + ethtool_link_ksettings_add_link_mode(ks, supported, + 10000baseT_Full); + break; + case IONIC_XCVR_PID_SFP_1000BASE_T: + ethtool_link_ksettings_add_link_mode(ks, supported, + 1000baseT_Full); + break; case IONIC_XCVR_PID_UNKNOWN: /* This means there's no module plugged in */ break; @@ -264,12 +270,10 @@ static int ionic_set_link_ksettings(struct net_device *netdev, const struct ethtool_link_ksettings *ks) { struct ionic_lif *lif = netdev_priv(netdev); + struct ionic_dev *idev = &lif->ionic->idev; struct ionic *ionic = lif->ionic; - struct ionic_dev *idev; int err = 0; - idev = &lif->ionic->idev; - /* set autoneg */ if (ks->base.autoneg != idev->port_info->config.an_enable) { mutex_lock(&ionic->dev_cmd_lock); diff --git a/drivers/net/ethernet/pensando/ionic/ionic_if.h b/drivers/net/ethernet/pensando/ionic/ionic_if.h index 31ccfcdc2b0a..88210142395d 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_if.h +++ b/drivers/net/ethernet/pensando/ionic/ionic_if.h @@ -320,7 +320,7 @@ struct ionic_lif_identify_comp { /** * enum ionic_lif_capability - LIF capabilities * @IONIC_LIF_CAP_ETH: LIF supports Ethernet - * @IONIC_LIF_CAP_RDMA: LIF support RDMA + * @IONIC_LIF_CAP_RDMA: LIF supports RDMA */ enum ionic_lif_capability { IONIC_LIF_CAP_ETH = BIT(0), @@ -404,7 +404,7 @@ union ionic_lif_config { * @max_ucast_filters: Number of perfect unicast addresses supported * @max_mcast_filters: Number of perfect multicast addresses supported * @min_frame_size: Minimum size of frames to be sent - * @max_frame_size: Maximim size of frames to be sent + * @max_frame_size: Maximum size of frames to be sent * @config: LIF config struct with features, mtu, mac, q counts * * @rdma: RDMA identify structure @@ -692,7 +692,7 @@ enum ionic_txq_desc_opcode { * checksums are also updated. * * IONIC_TXQ_DESC_OPCODE_TSO: - * Device preforms TCP segmentation offload + * Device performs TCP segmentation offload * (TSO). @hdr_len is the number of bytes * to the end of TCP header (the offset to * the TCP payload). @mss is the desired @@ -982,13 +982,13 @@ struct ionic_rxq_comp { }; enum ionic_pkt_type { - IONIC_PKT_TYPE_NON_IP = 0x000, - IONIC_PKT_TYPE_IPV4 = 0x001, - IONIC_PKT_TYPE_IPV4_TCP = 0x003, - IONIC_PKT_TYPE_IPV4_UDP = 0x005, - IONIC_PKT_TYPE_IPV6 = 0x008, - IONIC_PKT_TYPE_IPV6_TCP = 0x018, - IONIC_PKT_TYPE_IPV6_UDP = 0x028, + IONIC_PKT_TYPE_NON_IP = 0x00, + IONIC_PKT_TYPE_IPV4 = 0x01, + IONIC_PKT_TYPE_IPV4_TCP = 0x03, + IONIC_PKT_TYPE_IPV4_UDP = 0x05, + IONIC_PKT_TYPE_IPV6 = 0x08, + IONIC_PKT_TYPE_IPV6_TCP = 0x18, + IONIC_PKT_TYPE_IPV6_UDP = 0x28, /* below types are only used if encap offloads are enabled on lif */ IONIC_PKT_TYPE_ENCAP_NON_IP = 0x40, IONIC_PKT_TYPE_ENCAP_IPV4 = 0x41, @@ -1111,6 +1111,8 @@ enum ionic_xcvr_pid { IONIC_XCVR_PID_QSFP_100G_CWDM4 = 69, IONIC_XCVR_PID_QSFP_100G_PSM4 = 70, IONIC_XCVR_PID_SFP_25GBASE_ACC = 71, + IONIC_XCVR_PID_SFP_10GBASE_T = 72, + IONIC_XCVR_PID_SFP_1000BASE_T = 73, }; /** @@ -1331,7 +1333,7 @@ enum ionic_stats_ctl_cmd { * @IONIC_PORT_ATTR_STATE: Port state attribute * @IONIC_PORT_ATTR_SPEED: Port speed attribute * @IONIC_PORT_ATTR_MTU: Port MTU attribute - * @IONIC_PORT_ATTR_AUTONEG: Port autonegotation attribute + * @IONIC_PORT_ATTR_AUTONEG: Port autonegotiation attribute * @IONIC_PORT_ATTR_FEC: Port FEC attribute * @IONIC_PORT_ATTR_PAUSE: Port pause attribute * @IONIC_PORT_ATTR_LOOPBACK: Port loopback attribute @@ -1951,8 +1953,8 @@ enum ionic_qos_sched_type { * @pfc_cos: Priority-Flow Control class of service * @dwrr_weight: QoS class scheduling weight * @strict_rlmt: Rate limit for strict priority scheduling - * @rw_dot1q_pcp: Rewrite dot1q pcp to this value (valid iff F_RW_DOT1Q_PCP) - * @rw_ip_dscp: Rewrite ip dscp to this value (valid iff F_RW_IP_DSCP) + * @rw_dot1q_pcp: Rewrite dot1q pcp to value (valid iff F_RW_DOT1Q_PCP) + * @rw_ip_dscp: Rewrite ip dscp to value (valid iff F_RW_IP_DSCP) * @dot1q_pcp: Dot1q pcp value * @ndscp: Number of valid dscp values in the ip_dscp field * @ip_dscp: IP dscp values diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.c b/drivers/net/ethernet/pensando/ionic/ionic_lif.c index 11140915c2da..889d234e2ffa 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_lif.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.c @@ -120,23 +120,34 @@ static void ionic_link_status_check(struct ionic_lif *lif) if (!test_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state)) return; + /* Don't put carrier back up if we're in a broken state */ + if (test_bit(IONIC_LIF_F_BROKEN, lif->state)) { + clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state); + return; + } + link_status = le16_to_cpu(lif->info->status.link_status); link_up = link_status == IONIC_PORT_OPER_STATUS_UP; if (link_up) { - if (lif->netdev->flags & IFF_UP && netif_running(lif->netdev)) { + int err = 0; + + if (netdev->flags & IFF_UP && netif_running(netdev)) { mutex_lock(&lif->queue_lock); - ionic_start_queues(lif); + err = ionic_start_queues(lif); + if (err) { + netdev_err(lif->netdev, + "Failed to start queues: %d\n", err); + set_bit(IONIC_LIF_F_BROKEN, lif->state); + netif_carrier_off(lif->netdev); + } mutex_unlock(&lif->queue_lock); } - if (!netif_carrier_ok(netdev)) { - u32 link_speed; - + if (!err && !netif_carrier_ok(netdev)) { ionic_port_identify(lif->ionic); - link_speed = le32_to_cpu(lif->info->status.link_speed); netdev_info(netdev, "Link up - %d Gbps\n", - link_speed / 1000); + le32_to_cpu(lif->info->status.link_speed) / 1000); netif_carrier_on(netdev); } } else { @@ -145,7 +156,7 @@ static void ionic_link_status_check(struct ionic_lif *lif) netif_carrier_off(netdev); } - if (lif->netdev->flags & IFF_UP && netif_running(lif->netdev)) { + if (netdev->flags & IFF_UP && netif_running(netdev)) { mutex_lock(&lif->queue_lock); ionic_stop_queues(lif); mutex_unlock(&lif->queue_lock); @@ -382,6 +393,8 @@ static void ionic_qcq_free(struct ionic_lif *lif, struct ionic_qcq *qcq) static void ionic_qcqs_free(struct ionic_lif *lif) { struct device *dev = lif->ionic->dev; + struct ionic_qcq *adminqcq; + unsigned long irqflags; if (lif->notifyqcq) { ionic_qcq_free(lif, lif->notifyqcq); @@ -390,9 +403,14 @@ static void ionic_qcqs_free(struct ionic_lif *lif) } if (lif->adminqcq) { - ionic_qcq_free(lif, lif->adminqcq); - devm_kfree(dev, lif->adminqcq); + spin_lock_irqsave(&lif->adminq_lock, irqflags); + adminqcq = READ_ONCE(lif->adminqcq); lif->adminqcq = NULL; + spin_unlock_irqrestore(&lif->adminq_lock, irqflags); + if (adminqcq) { + ionic_qcq_free(lif, adminqcq); + devm_kfree(dev, adminqcq); + } } if (lif->rxqcqs) { @@ -495,6 +513,7 @@ static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type, goto err_out; } + new->q.dev = dev; new->flags = flags; new->q.info = devm_kcalloc(dev, num_descs, sizeof(*new->q.info), @@ -506,6 +525,7 @@ static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type, } new->q.type = type; + new->q.max_sg_elems = lif->qtype_info[type].max_sg_elems; err = ionic_q_init(lif, idev, &new->q, index, name, num_descs, desc_size, sg_desc_size, pid); @@ -716,10 +736,8 @@ static int ionic_lif_txq_init(struct ionic_lif *lif, struct ionic_qcq *qcq) unsigned int intr_index; int err; - if (qcq->flags & IONIC_QCQ_F_INTR) - intr_index = qcq->intr.index; - else - intr_index = lif->rxqcqs[q->index]->intr.index; + intr_index = qcq->intr.index; + ctx.cmd.q_init.intr_index = cpu_to_le16(intr_index); dev_dbg(dev, "txq_init.pid %d\n", ctx.cmd.q_init.pid); @@ -837,7 +855,7 @@ static bool ionic_notifyq_service(struct ionic_cq *cq, switch (le16_to_cpu(comp->event.ecode)) { case IONIC_EVENT_LINK_CHANGE: - ionic_link_status_check_request(lif, false); + ionic_link_status_check_request(lif, CAN_NOT_SLEEP); break; case IONIC_EVENT_RESET: work = kzalloc(sizeof(*work), GFP_ATOMIC); @@ -875,6 +893,7 @@ static int ionic_adminq_napi(struct napi_struct *napi, int budget) struct ionic_intr_info *intr = napi_to_cq(napi)->bound_intr; struct ionic_lif *lif = napi_to_cq(napi)->lif; struct ionic_dev *idev = &lif->ionic->idev; + unsigned long irqflags; unsigned int flags = 0; int n_work = 0; int a_work = 0; @@ -884,14 +903,16 @@ static int ionic_adminq_napi(struct napi_struct *napi, int budget) n_work = ionic_cq_service(&lif->notifyqcq->cq, budget, ionic_notifyq_service, NULL, NULL); + spin_lock_irqsave(&lif->adminq_lock, irqflags); if (lif->adminqcq && lif->adminqcq->flags & IONIC_QCQ_F_INITED) a_work = ionic_cq_service(&lif->adminqcq->cq, budget, ionic_adminq_service, NULL, NULL); + spin_unlock_irqrestore(&lif->adminq_lock, irqflags); work_done = max(n_work, a_work); if (work_done < budget && napi_complete_done(napi, work_done)) { flags |= IONIC_INTR_CRED_UNMASK; - lif->adminqcq->cq.bound_intr->rearm_count++; + intr->rearm_count++; } if (work_done || flags) { @@ -1441,7 +1462,7 @@ static int ionic_start_queues_reconfig(struct ionic_lif *lif) */ err = ionic_txrx_init(lif); mutex_unlock(&lif->queue_lock); - ionic_link_status_check_request(lif, true); + ionic_link_status_check_request(lif, CAN_SLEEP); netif_device_attach(lif->netdev); return err; @@ -1480,7 +1501,8 @@ static void ionic_tx_timeout_work(struct work_struct *ws) { struct ionic_lif *lif = container_of(ws, struct ionic_lif, tx_timeout_work); - netdev_info(lif->netdev, "Tx Timeout recovery\n"); + if (test_bit(IONIC_LIF_F_FW_RESET, lif->state)) + return; /* if we were stopped before this scheduled job was launched, * don't bother the queues as they are already stopped. @@ -1496,6 +1518,7 @@ static void ionic_tx_timeout(struct net_device *netdev, unsigned int txqueue) { struct ionic_lif *lif = netdev_priv(netdev); + netdev_info(lif->netdev, "Tx Timeout triggered - txq %d\n", txqueue); schedule_work(&lif->tx_timeout_work); } @@ -1837,6 +1860,12 @@ static int ionic_start_queues(struct ionic_lif *lif) { int err; + if (test_bit(IONIC_LIF_F_BROKEN, lif->state)) + return -EIO; + + if (test_bit(IONIC_LIF_F_FW_RESET, lif->state)) + return -EBUSY; + if (test_and_set_bit(IONIC_LIF_F_UP, lif->state)) return 0; @@ -1855,13 +1884,17 @@ static int ionic_open(struct net_device *netdev) struct ionic_lif *lif = netdev_priv(netdev); int err; + /* If recovering from a broken state, clear the bit and we'll try again */ + if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state)) + netdev_info(netdev, "clearing broken state\n"); + err = ionic_txrx_alloc(lif); if (err) return err; err = ionic_txrx_init(lif); if (err) - goto err_out; + goto err_txrx_free; err = netif_set_real_num_tx_queues(netdev, lif->nxqs); if (err) @@ -1882,7 +1915,7 @@ static int ionic_open(struct net_device *netdev) err_txrx_deinit: ionic_txrx_deinit(lif); -err_out: +err_txrx_free: ionic_txrx_free(lif); return err; } @@ -2202,6 +2235,9 @@ static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b) swap(a->cq_base, b->cq_base); swap(a->cq_base_pa, b->cq_base_pa); swap(a->cq_size, b->cq_size); + + ionic_debugfs_del_qcq(a); + ionic_debugfs_add_qcq(a->q.lif, a); } int ionic_reconfigure_queues(struct ionic_lif *lif, @@ -2351,7 +2387,7 @@ int ionic_reconfigure_queues(struct ionic_lif *lif, swap(lif->nxqs, qparam->nxqs); err_out_reinit_unlock: - /* re-init the queues, but don't loose an error code */ + /* re-init the queues, but don't lose an error code */ if (err) ionic_start_queues_reconfig(lif); else @@ -2450,7 +2486,6 @@ int ionic_lif_alloc(struct ionic *ionic) lif->index = 0; lif->ntxq_descs = IONIC_DEF_TXRX_DESC; lif->nrxq_descs = IONIC_DEF_TXRX_DESC; - lif->tx_budget = IONIC_TX_BUDGET_DEFAULT; /* Convert the default coalesce value to actual hw resolution */ lif->rx_coalesce_usecs = IONIC_ITR_COAL_USEC_DEFAULT; @@ -2601,7 +2636,7 @@ static void ionic_lif_handle_fw_up(struct ionic_lif *lif) } clear_bit(IONIC_LIF_F_FW_RESET, lif->state); - ionic_link_status_check_request(lif, true); + ionic_link_status_check_request(lif, CAN_SLEEP); netif_device_attach(lif->netdev); dev_info(ionic->dev, "FW Up: LIFs restarted\n"); @@ -2972,7 +3007,7 @@ int ionic_lif_register(struct ionic_lif *lif) return err; } - ionic_link_status_check_request(lif, true); + ionic_link_status_check_request(lif, CAN_SLEEP); lif->registered = true; ionic_lif_set_netdev_info(lif); diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.h b/drivers/net/ethernet/pensando/ionic/ionic_lif.h index 563dba384a53..be5cc89b2bd9 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_lif.h +++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.h @@ -139,6 +139,7 @@ enum ionic_lif_state_flags { IONIC_LIF_F_LINK_CHECK_REQUESTED, IONIC_LIF_F_FW_RESET, IONIC_LIF_F_SPLIT_INTR, + IONIC_LIF_F_BROKEN, IONIC_LIF_F_TX_DIM_INTR, IONIC_LIF_F_RX_DIM_INTR, @@ -159,16 +160,11 @@ struct ionic_qtype_info { #define IONIC_LIF_NAME_MAX_SZ 32 struct ionic_lif { - char name[IONIC_LIF_NAME_MAX_SZ]; - struct list_head list; struct net_device *netdev; DECLARE_BITMAP(state, IONIC_LIF_F_STATE_SIZE); struct ionic *ionic; - bool registered; unsigned int index; unsigned int hw_index; - unsigned int kern_pid; - u64 __iomem *kern_dbpage; struct mutex queue_lock; /* lock for queue structures */ spinlock_t adminq_lock; /* lock for AdminQ operations */ struct ionic_qcq *adminqcq; @@ -177,20 +173,25 @@ struct ionic_lif { struct ionic_tx_stats *txqstats; struct ionic_qcq **rxqcqs; struct ionic_rx_stats *rxqstats; + struct ionic_deferred deferred; + struct work_struct tx_timeout_work; u64 last_eid; + unsigned int kern_pid; + u64 __iomem *kern_dbpage; unsigned int neqs; unsigned int nxqs; unsigned int ntxq_descs; unsigned int nrxq_descs; u32 rx_copybreak; - u32 tx_budget; unsigned int rx_mode; u64 hw_features; + bool registered; bool mc_overflow; - unsigned int nmcast; bool uc_overflow; u16 lif_type; + unsigned int nmcast; unsigned int nucast; + char name[IONIC_LIF_NAME_MAX_SZ]; union ionic_lif_identity *identity; struct ionic_lif_info *info; @@ -205,16 +206,14 @@ struct ionic_lif { u32 rss_ind_tbl_sz; struct ionic_rx_filters rx_filters; - struct ionic_deferred deferred; - unsigned long *dbid_inuse; - unsigned int dbid_count; - struct dentry *dentry; u32 rx_coalesce_usecs; /* what the user asked for */ u32 rx_coalesce_hw; /* what the hw is using */ u32 tx_coalesce_usecs; /* what the user asked for */ u32 tx_coalesce_hw; /* what the hw is using */ + unsigned long *dbid_inuse; + unsigned int dbid_count; - struct work_struct tx_timeout_work; + struct dentry *dentry; }; struct ionic_queue_params { diff --git a/drivers/net/ethernet/pensando/ionic/ionic_main.c b/drivers/net/ethernet/pensando/ionic/ionic_main.c index fbc57de6683e..c4b2906a2ae6 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_main.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_main.c @@ -187,10 +187,17 @@ static const char *ionic_opcode_to_str(enum ionic_cmd_opcode opcode) static void ionic_adminq_flush(struct ionic_lif *lif) { - struct ionic_queue *q = &lif->adminqcq->q; struct ionic_desc_info *desc_info; + unsigned long irqflags; + struct ionic_queue *q; + + spin_lock_irqsave(&lif->adminq_lock, irqflags); + if (!lif->adminqcq) { + spin_unlock_irqrestore(&lif->adminq_lock, irqflags); + return; + } - spin_lock(&lif->adminq_lock); + q = &lif->adminqcq->q; while (q->tail_idx != q->head_idx) { desc_info = &q->info[q->tail_idx]; @@ -199,7 +206,7 @@ static void ionic_adminq_flush(struct ionic_lif *lif) desc_info->cb_arg = NULL; q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); } - spin_unlock(&lif->adminq_lock); + spin_unlock_irqrestore(&lif->adminq_lock, irqflags); } static int ionic_adminq_check_err(struct ionic_lif *lif, @@ -234,17 +241,15 @@ static void ionic_adminq_cb(struct ionic_queue *q, { struct ionic_admin_ctx *ctx = cb_arg; struct ionic_admin_comp *comp; - struct device *dev; if (!ctx) return; comp = cq_info->cq_desc; - dev = &q->lif->netdev->dev; memcpy(&ctx->comp, comp, sizeof(*comp)); - dev_dbg(dev, "comp admin queue command:\n"); + dev_dbg(q->dev, "comp admin queue command:\n"); dynamic_hex_dump("comp ", DUMP_PREFIX_OFFSET, 16, 1, &ctx->comp, sizeof(ctx->comp), true); @@ -254,15 +259,18 @@ static void ionic_adminq_cb(struct ionic_queue *q, static int ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx) { struct ionic_desc_info *desc_info; + unsigned long irqflags; struct ionic_queue *q; int err = 0; - if (!lif->adminqcq) + spin_lock_irqsave(&lif->adminq_lock, irqflags); + if (!lif->adminqcq) { + spin_unlock_irqrestore(&lif->adminq_lock, irqflags); return -EIO; + } q = &lif->adminqcq->q; - spin_lock(&lif->adminq_lock); if (!ionic_q_has_space(q, 1)) { err = -ENOSPC; goto err_out; @@ -282,7 +290,7 @@ static int ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx) ionic_q_post(q, true, ionic_adminq_cb, ctx); err_out: - spin_unlock(&lif->adminq_lock); + spin_unlock_irqrestore(&lif->adminq_lock, irqflags); return err; } diff --git a/drivers/net/ethernet/pensando/ionic/ionic_stats.c b/drivers/net/ethernet/pensando/ionic/ionic_stats.c index 6ae75b771a15..308b4ac6c57b 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_stats.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_stats.c @@ -246,98 +246,73 @@ static u64 ionic_sw_stats_get_count(struct ionic_lif *lif) return total; } +static void ionic_sw_stats_get_tx_strings(struct ionic_lif *lif, u8 **buf, + int q_num) +{ + int i; + + for (i = 0; i < IONIC_NUM_TX_STATS; i++) + ethtool_sprintf(buf, "tx_%d_%s", q_num, + ionic_tx_stats_desc[i].name); + + if (!test_bit(IONIC_LIF_F_UP, lif->state) || + !test_bit(IONIC_LIF_F_SW_DEBUG_STATS, lif->state)) + return; + + for (i = 0; i < IONIC_NUM_TX_Q_STATS; i++) + ethtool_sprintf(buf, "txq_%d_%s", q_num, + ionic_txq_stats_desc[i].name); + for (i = 0; i < IONIC_NUM_DBG_CQ_STATS; i++) + ethtool_sprintf(buf, "txq_%d_cq_%s", q_num, + ionic_dbg_cq_stats_desc[i].name); + for (i = 0; i < IONIC_NUM_DBG_INTR_STATS; i++) + ethtool_sprintf(buf, "txq_%d_intr_%s", q_num, + ionic_dbg_intr_stats_desc[i].name); + for (i = 0; i < IONIC_MAX_NUM_SG_CNTR; i++) + ethtool_sprintf(buf, "txq_%d_sg_cntr_%d", q_num, i); +} + +static void ionic_sw_stats_get_rx_strings(struct ionic_lif *lif, u8 **buf, + int q_num) +{ + int i; + + for (i = 0; i < IONIC_NUM_RX_STATS; i++) + ethtool_sprintf(buf, "rx_%d_%s", q_num, + ionic_rx_stats_desc[i].name); + + if (!test_bit(IONIC_LIF_F_UP, lif->state) || + !test_bit(IONIC_LIF_F_SW_DEBUG_STATS, lif->state)) + return; + + for (i = 0; i < IONIC_NUM_DBG_CQ_STATS; i++) + ethtool_sprintf(buf, "rxq_%d_cq_%s", q_num, + ionic_dbg_cq_stats_desc[i].name); + for (i = 0; i < IONIC_NUM_DBG_INTR_STATS; i++) + ethtool_sprintf(buf, "rxq_%d_intr_%s", q_num, + ionic_dbg_intr_stats_desc[i].name); + for (i = 0; i < IONIC_NUM_DBG_NAPI_STATS; i++) + ethtool_sprintf(buf, "rxq_%d_napi_%s", q_num, + ionic_dbg_napi_stats_desc[i].name); + for (i = 0; i < IONIC_MAX_NUM_NAPI_CNTR; i++) + ethtool_sprintf(buf, "rxq_%d_napi_work_done_%d", q_num, i); +} + static void ionic_sw_stats_get_strings(struct ionic_lif *lif, u8 **buf) { int i, q_num; - for (i = 0; i < IONIC_NUM_LIF_STATS; i++) { - snprintf(*buf, ETH_GSTRING_LEN, ionic_lif_stats_desc[i].name); - *buf += ETH_GSTRING_LEN; - } + for (i = 0; i < IONIC_NUM_LIF_STATS; i++) + ethtool_sprintf(buf, ionic_lif_stats_desc[i].name); - for (i = 0; i < IONIC_NUM_PORT_STATS; i++) { - snprintf(*buf, ETH_GSTRING_LEN, - ionic_port_stats_desc[i].name); - *buf += ETH_GSTRING_LEN; - } + for (i = 0; i < IONIC_NUM_PORT_STATS; i++) + ethtool_sprintf(buf, ionic_port_stats_desc[i].name); - for (q_num = 0; q_num < MAX_Q(lif); q_num++) { - for (i = 0; i < IONIC_NUM_TX_STATS; i++) { - snprintf(*buf, ETH_GSTRING_LEN, "tx_%d_%s", - q_num, ionic_tx_stats_desc[i].name); - *buf += ETH_GSTRING_LEN; - } + for (q_num = 0; q_num < MAX_Q(lif); q_num++) + ionic_sw_stats_get_tx_strings(lif, buf, q_num); - if (test_bit(IONIC_LIF_F_UP, lif->state) && - test_bit(IONIC_LIF_F_SW_DEBUG_STATS, lif->state)) { - for (i = 0; i < IONIC_NUM_TX_Q_STATS; i++) { - snprintf(*buf, ETH_GSTRING_LEN, - "txq_%d_%s", - q_num, - ionic_txq_stats_desc[i].name); - *buf += ETH_GSTRING_LEN; - } - for (i = 0; i < IONIC_NUM_DBG_CQ_STATS; i++) { - snprintf(*buf, ETH_GSTRING_LEN, - "txq_%d_cq_%s", - q_num, - ionic_dbg_cq_stats_desc[i].name); - *buf += ETH_GSTRING_LEN; - } - for (i = 0; i < IONIC_NUM_DBG_INTR_STATS; i++) { - snprintf(*buf, ETH_GSTRING_LEN, - "txq_%d_intr_%s", - q_num, - ionic_dbg_intr_stats_desc[i].name); - *buf += ETH_GSTRING_LEN; - } - for (i = 0; i < IONIC_MAX_NUM_SG_CNTR; i++) { - snprintf(*buf, ETH_GSTRING_LEN, - "txq_%d_sg_cntr_%d", - q_num, i); - *buf += ETH_GSTRING_LEN; - } - } - } - for (q_num = 0; q_num < MAX_Q(lif); q_num++) { - for (i = 0; i < IONIC_NUM_RX_STATS; i++) { - snprintf(*buf, ETH_GSTRING_LEN, - "rx_%d_%s", - q_num, ionic_rx_stats_desc[i].name); - *buf += ETH_GSTRING_LEN; - } - - if (test_bit(IONIC_LIF_F_UP, lif->state) && - test_bit(IONIC_LIF_F_SW_DEBUG_STATS, lif->state)) { - for (i = 0; i < IONIC_NUM_DBG_CQ_STATS; i++) { - snprintf(*buf, ETH_GSTRING_LEN, - "rxq_%d_cq_%s", - q_num, - ionic_dbg_cq_stats_desc[i].name); - *buf += ETH_GSTRING_LEN; - } - for (i = 0; i < IONIC_NUM_DBG_INTR_STATS; i++) { - snprintf(*buf, ETH_GSTRING_LEN, - "rxq_%d_intr_%s", - q_num, - ionic_dbg_intr_stats_desc[i].name); - *buf += ETH_GSTRING_LEN; - } - for (i = 0; i < IONIC_NUM_DBG_NAPI_STATS; i++) { - snprintf(*buf, ETH_GSTRING_LEN, - "rxq_%d_napi_%s", - q_num, - ionic_dbg_napi_stats_desc[i].name); - *buf += ETH_GSTRING_LEN; - } - for (i = 0; i < IONIC_MAX_NUM_NAPI_CNTR; i++) { - snprintf(*buf, ETH_GSTRING_LEN, - "rxq_%d_napi_work_done_%d", - q_num, i); - *buf += ETH_GSTRING_LEN; - } - } - } + for (q_num = 0; q_num < MAX_Q(lif); q_num++) + ionic_sw_stats_get_rx_strings(lif, buf, q_num); } static void ionic_sw_stats_get_values(struct ionic_lif *lif, u64 **buf) diff --git a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c index 162a1ff1e9d2..5985f7c504a9 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c @@ -10,12 +10,6 @@ #include "ionic_lif.h" #include "ionic_txrx.h" -static void ionic_rx_clean(struct ionic_queue *q, - struct ionic_desc_info *desc_info, - struct ionic_cq_info *cq_info, - void *cb_arg); - -static bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info); static bool ionic_tx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info); @@ -40,72 +34,149 @@ static inline struct netdev_queue *q_to_ndq(struct ionic_queue *q) return netdev_get_tx_queue(q->lif->netdev, q->index); } -static struct sk_buff *ionic_rx_skb_alloc(struct ionic_queue *q, - unsigned int len, bool frags) +static void ionic_rx_buf_reset(struct ionic_buf_info *buf_info) +{ + buf_info->page = NULL; + buf_info->page_offset = 0; + buf_info->dma_addr = 0; +} + +static int ionic_rx_page_alloc(struct ionic_queue *q, + struct ionic_buf_info *buf_info) { - struct ionic_lif *lif = q->lif; + struct net_device *netdev = q->lif->netdev; struct ionic_rx_stats *stats; - struct net_device *netdev; - struct sk_buff *skb; + struct device *dev; - netdev = lif->netdev; - stats = &q->lif->rxqstats[q->index]; + dev = q->dev; + stats = q_to_rx_stats(q); - if (frags) - skb = napi_get_frags(&q_to_qcq(q)->napi); - else - skb = netdev_alloc_skb_ip_align(netdev, len); + if (unlikely(!buf_info)) { + net_err_ratelimited("%s: %s invalid buf_info in alloc\n", + netdev->name, q->name); + return -EINVAL; + } - if (unlikely(!skb)) { - net_warn_ratelimited("%s: SKB alloc failed on %s!\n", - netdev->name, q->name); + buf_info->page = alloc_pages(IONIC_PAGE_GFP_MASK, 0); + if (unlikely(!buf_info->page)) { + net_err_ratelimited("%s: %s page alloc failed\n", + netdev->name, q->name); stats->alloc_err++; - return NULL; + return -ENOMEM; } + buf_info->page_offset = 0; - return skb; + buf_info->dma_addr = dma_map_page(dev, buf_info->page, buf_info->page_offset, + IONIC_PAGE_SIZE, DMA_FROM_DEVICE); + if (unlikely(dma_mapping_error(dev, buf_info->dma_addr))) { + __free_pages(buf_info->page, 0); + ionic_rx_buf_reset(buf_info); + net_err_ratelimited("%s: %s dma map failed\n", + netdev->name, q->name); + stats->dma_map_err++; + return -EIO; + } + + return 0; +} + +static void ionic_rx_page_free(struct ionic_queue *q, + struct ionic_buf_info *buf_info) +{ + struct net_device *netdev = q->lif->netdev; + struct device *dev = q->dev; + + if (unlikely(!buf_info)) { + net_err_ratelimited("%s: %s invalid buf_info in free\n", + netdev->name, q->name); + return; + } + + if (!buf_info->page) + return; + + dma_unmap_page(dev, buf_info->dma_addr, IONIC_PAGE_SIZE, DMA_FROM_DEVICE); + __free_pages(buf_info->page, 0); + ionic_rx_buf_reset(buf_info); +} + +static bool ionic_rx_buf_recycle(struct ionic_queue *q, + struct ionic_buf_info *buf_info, u32 used) +{ + u32 size; + + /* don't re-use pages allocated in low-mem condition */ + if (page_is_pfmemalloc(buf_info->page)) + return false; + + /* don't re-use buffers from non-local numa nodes */ + if (page_to_nid(buf_info->page) != numa_mem_id()) + return false; + + size = ALIGN(used, IONIC_PAGE_SPLIT_SZ); + buf_info->page_offset += size; + if (buf_info->page_offset >= IONIC_PAGE_SIZE) + return false; + + get_page(buf_info->page); + + return true; } static struct sk_buff *ionic_rx_frags(struct ionic_queue *q, struct ionic_desc_info *desc_info, - struct ionic_cq_info *cq_info) + struct ionic_rxq_comp *comp) { - struct ionic_rxq_comp *comp = cq_info->cq_desc; - struct device *dev = q->lif->ionic->dev; - struct ionic_page_info *page_info; + struct net_device *netdev = q->lif->netdev; + struct ionic_buf_info *buf_info; + struct ionic_rx_stats *stats; + struct device *dev = q->dev; struct sk_buff *skb; unsigned int i; u16 frag_len; u16 len; - page_info = &desc_info->pages[0]; + stats = q_to_rx_stats(q); + + buf_info = &desc_info->bufs[0]; len = le16_to_cpu(comp->len); - prefetch(page_address(page_info->page) + NET_IP_ALIGN); + prefetch(buf_info->page); - skb = ionic_rx_skb_alloc(q, len, true); - if (unlikely(!skb)) + skb = napi_get_frags(&q_to_qcq(q)->napi); + if (unlikely(!skb)) { + net_warn_ratelimited("%s: SKB alloc failed on %s!\n", + netdev->name, q->name); + stats->alloc_err++; return NULL; + } i = comp->num_sg_elems + 1; do { - if (unlikely(!page_info->page)) { - struct napi_struct *napi = &q_to_qcq(q)->napi; - - napi->skb = NULL; + if (unlikely(!buf_info->page)) { dev_kfree_skb(skb); return NULL; } - frag_len = min(len, (u16)PAGE_SIZE); + frag_len = min_t(u16, len, IONIC_PAGE_SIZE - buf_info->page_offset); len -= frag_len; - dma_unmap_page(dev, dma_unmap_addr(page_info, dma_addr), - PAGE_SIZE, DMA_FROM_DEVICE); + dma_sync_single_for_cpu(dev, + buf_info->dma_addr + buf_info->page_offset, + frag_len, DMA_FROM_DEVICE); + skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, - page_info->page, 0, frag_len, PAGE_SIZE); - page_info->page = NULL; - page_info++; + buf_info->page, buf_info->page_offset, frag_len, + IONIC_PAGE_SIZE); + + if (!ionic_rx_buf_recycle(q, buf_info, frag_len)) { + dma_unmap_page(dev, buf_info->dma_addr, + IONIC_PAGE_SIZE, DMA_FROM_DEVICE); + ionic_rx_buf_reset(buf_info); + } + + buf_info++; + i--; } while (i > 0); @@ -114,30 +185,37 @@ static struct sk_buff *ionic_rx_frags(struct ionic_queue *q, static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q, struct ionic_desc_info *desc_info, - struct ionic_cq_info *cq_info) + struct ionic_rxq_comp *comp) { - struct ionic_rxq_comp *comp = cq_info->cq_desc; - struct device *dev = q->lif->ionic->dev; - struct ionic_page_info *page_info; + struct net_device *netdev = q->lif->netdev; + struct ionic_buf_info *buf_info; + struct ionic_rx_stats *stats; + struct device *dev = q->dev; struct sk_buff *skb; u16 len; - page_info = &desc_info->pages[0]; + stats = q_to_rx_stats(q); + + buf_info = &desc_info->bufs[0]; len = le16_to_cpu(comp->len); - skb = ionic_rx_skb_alloc(q, len, false); - if (unlikely(!skb)) + skb = napi_alloc_skb(&q_to_qcq(q)->napi, len); + if (unlikely(!skb)) { + net_warn_ratelimited("%s: SKB alloc failed on %s!\n", + netdev->name, q->name); + stats->alloc_err++; return NULL; + } - if (unlikely(!page_info->page)) { + if (unlikely(!buf_info->page)) { dev_kfree_skb(skb); return NULL; } - dma_sync_single_for_cpu(dev, dma_unmap_addr(page_info, dma_addr), + dma_sync_single_for_cpu(dev, buf_info->dma_addr + buf_info->page_offset, len, DMA_FROM_DEVICE); - skb_copy_to_linear_data(skb, page_address(page_info->page), len); - dma_sync_single_for_device(dev, dma_unmap_addr(page_info, dma_addr), + skb_copy_to_linear_data(skb, page_address(buf_info->page) + buf_info->page_offset, len); + dma_sync_single_for_device(dev, buf_info->dma_addr + buf_info->page_offset, len, DMA_FROM_DEVICE); skb_put(skb, len); @@ -151,14 +229,13 @@ static void ionic_rx_clean(struct ionic_queue *q, struct ionic_cq_info *cq_info, void *cb_arg) { - struct ionic_rxq_comp *comp = cq_info->cq_desc; + struct ionic_rxq_comp *comp = cq_info->rxcq; + struct net_device *netdev = q->lif->netdev; struct ionic_qcq *qcq = q_to_qcq(q); struct ionic_rx_stats *stats; - struct net_device *netdev; struct sk_buff *skb; stats = q_to_rx_stats(q); - netdev = q->lif->netdev; if (comp->status) { stats->dropped++; @@ -169,9 +246,9 @@ static void ionic_rx_clean(struct ionic_queue *q, stats->bytes += le16_to_cpu(comp->len); if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak) - skb = ionic_rx_copybreak(q, desc_info, cq_info); + skb = ionic_rx_copybreak(q, desc_info, comp); else - skb = ionic_rx_frags(q, desc_info, cq_info); + skb = ionic_rx_frags(q, desc_info, comp); if (unlikely(!skb)) { stats->dropped++; @@ -227,7 +304,7 @@ static void ionic_rx_clean(struct ionic_queue *q, static bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info) { - struct ionic_rxq_comp *comp = cq_info->cq_desc; + struct ionic_rxq_comp *comp = cq_info->rxcq; struct ionic_queue *q = cq->bound_q; struct ionic_desc_info *desc_info; @@ -253,138 +330,75 @@ static bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info) return true; } -static int ionic_rx_page_alloc(struct ionic_queue *q, - struct ionic_page_info *page_info) -{ - struct ionic_lif *lif = q->lif; - struct ionic_rx_stats *stats; - struct net_device *netdev; - struct device *dev; - - netdev = lif->netdev; - dev = lif->ionic->dev; - stats = q_to_rx_stats(q); - - if (unlikely(!page_info)) { - net_err_ratelimited("%s: %s invalid page_info in alloc\n", - netdev->name, q->name); - return -EINVAL; - } - - page_info->page = dev_alloc_page(); - if (unlikely(!page_info->page)) { - net_err_ratelimited("%s: %s page alloc failed\n", - netdev->name, q->name); - stats->alloc_err++; - return -ENOMEM; - } - - page_info->dma_addr = dma_map_page(dev, page_info->page, 0, PAGE_SIZE, - DMA_FROM_DEVICE); - if (unlikely(dma_mapping_error(dev, page_info->dma_addr))) { - put_page(page_info->page); - page_info->dma_addr = 0; - page_info->page = NULL; - net_err_ratelimited("%s: %s dma map failed\n", - netdev->name, q->name); - stats->dma_map_err++; - return -EIO; - } - - return 0; -} - -static void ionic_rx_page_free(struct ionic_queue *q, - struct ionic_page_info *page_info) -{ - struct ionic_lif *lif = q->lif; - struct net_device *netdev; - struct device *dev; - - netdev = lif->netdev; - dev = lif->ionic->dev; - - if (unlikely(!page_info)) { - net_err_ratelimited("%s: %s invalid page_info in free\n", - netdev->name, q->name); - return; - } - - if (unlikely(!page_info->page)) { - net_err_ratelimited("%s: %s invalid page in free\n", - netdev->name, q->name); - return; - } - - dma_unmap_page(dev, page_info->dma_addr, PAGE_SIZE, DMA_FROM_DEVICE); - - put_page(page_info->page); - page_info->dma_addr = 0; - page_info->page = NULL; -} - void ionic_rx_fill(struct ionic_queue *q) { struct net_device *netdev = q->lif->netdev; struct ionic_desc_info *desc_info; - struct ionic_page_info *page_info; struct ionic_rxq_sg_desc *sg_desc; struct ionic_rxq_sg_elem *sg_elem; + struct ionic_buf_info *buf_info; struct ionic_rxq_desc *desc; unsigned int remain_len; - unsigned int seg_len; + unsigned int frag_len; unsigned int nfrags; unsigned int i, j; unsigned int len; len = netdev->mtu + ETH_HLEN + VLAN_HLEN; - nfrags = round_up(len, PAGE_SIZE) / PAGE_SIZE; for (i = ionic_q_space_avail(q); i; i--) { + nfrags = 0; remain_len = len; desc_info = &q->info[q->head_idx]; desc = desc_info->desc; - sg_desc = desc_info->sg_desc; - page_info = &desc_info->pages[0]; - - if (page_info->page) { /* recycle the buffer */ - ionic_rxq_post(q, false, ionic_rx_clean, NULL); - continue; - } + buf_info = &desc_info->bufs[0]; - /* fill main descriptor - pages[0] */ - desc->opcode = (nfrags > 1) ? IONIC_RXQ_DESC_OPCODE_SG : - IONIC_RXQ_DESC_OPCODE_SIMPLE; - desc_info->npages = nfrags; - if (unlikely(ionic_rx_page_alloc(q, page_info))) { - desc->addr = 0; - desc->len = 0; - return; + if (!buf_info->page) { /* alloc a new buffer? */ + if (unlikely(ionic_rx_page_alloc(q, buf_info))) { + desc->addr = 0; + desc->len = 0; + return; + } } - desc->addr = cpu_to_le64(page_info->dma_addr); - seg_len = min_t(unsigned int, PAGE_SIZE, len); - desc->len = cpu_to_le16(seg_len); - remain_len -= seg_len; - page_info++; - /* fill sg descriptors - pages[1..n] */ - for (j = 0; j < nfrags - 1; j++) { - if (page_info->page) /* recycle the sg buffer */ - continue; + /* fill main descriptor - buf[0] */ + desc->addr = cpu_to_le64(buf_info->dma_addr + buf_info->page_offset); + frag_len = min_t(u16, len, IONIC_PAGE_SIZE - buf_info->page_offset); + desc->len = cpu_to_le16(frag_len); + remain_len -= frag_len; + buf_info++; + nfrags++; + /* fill sg descriptors - buf[1..n] */ + sg_desc = desc_info->sg_desc; + for (j = 0; remain_len > 0 && j < q->max_sg_elems; j++) { sg_elem = &sg_desc->elems[j]; - if (unlikely(ionic_rx_page_alloc(q, page_info))) { - sg_elem->addr = 0; - sg_elem->len = 0; - return; + if (!buf_info->page) { /* alloc a new sg buffer? */ + if (unlikely(ionic_rx_page_alloc(q, buf_info))) { + sg_elem->addr = 0; + sg_elem->len = 0; + return; + } } - sg_elem->addr = cpu_to_le64(page_info->dma_addr); - seg_len = min_t(unsigned int, PAGE_SIZE, remain_len); - sg_elem->len = cpu_to_le16(seg_len); - remain_len -= seg_len; - page_info++; + + sg_elem->addr = cpu_to_le64(buf_info->dma_addr + buf_info->page_offset); + frag_len = min_t(u16, remain_len, IONIC_PAGE_SIZE - buf_info->page_offset); + sg_elem->len = cpu_to_le16(frag_len); + remain_len -= frag_len; + buf_info++; + nfrags++; + } + + /* clear end sg element as a sentinel */ + if (j < q->max_sg_elems) { + sg_elem = &sg_desc->elems[j]; + memset(sg_elem, 0, sizeof(*sg_elem)); } + desc->opcode = (nfrags > 1) ? IONIC_RXQ_DESC_OPCODE_SG : + IONIC_RXQ_DESC_OPCODE_SIMPLE; + desc_info->nbufs = nfrags; + ionic_rxq_post(q, false, ionic_rx_clean, NULL); } @@ -395,21 +409,24 @@ void ionic_rx_fill(struct ionic_queue *q) void ionic_rx_empty(struct ionic_queue *q) { struct ionic_desc_info *desc_info; - struct ionic_page_info *page_info; + struct ionic_buf_info *buf_info; unsigned int i, j; for (i = 0; i < q->num_descs; i++) { desc_info = &q->info[i]; for (j = 0; j < IONIC_RX_MAX_SG_ELEMS + 1; j++) { - page_info = &desc_info->pages[j]; - if (page_info->page) - ionic_rx_page_free(q, page_info); + buf_info = &desc_info->bufs[j]; + if (buf_info->page) + ionic_rx_page_free(q, buf_info); } - desc_info->npages = 0; + desc_info->nbufs = 0; desc_info->cb = NULL; desc_info->cb_arg = NULL; } + + q->head_idx = 0; + q->tail_idx = 0; } static void ionic_dim_update(struct ionic_qcq *qcq) @@ -525,7 +542,7 @@ int ionic_txrx_napi(struct napi_struct *napi, int budget) idev = &lif->ionic->idev; txcq = &lif->txqcqs[qi]->cq; - tx_work_done = ionic_cq_service(txcq, lif->tx_budget, + tx_work_done = ionic_cq_service(txcq, IONIC_TX_BUDGET_DEFAULT, ionic_tx_service, NULL, NULL); rx_work_done = ionic_cq_service(rxcq, budget, @@ -558,7 +575,7 @@ static dma_addr_t ionic_tx_map_single(struct ionic_queue *q, void *data, size_t len) { struct ionic_tx_stats *stats = q_to_tx_stats(q); - struct device *dev = q->lif->ionic->dev; + struct device *dev = q->dev; dma_addr_t dma_addr; dma_addr = dma_map_single(dev, data, len, DMA_TO_DEVICE); @@ -576,7 +593,7 @@ static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q, size_t offset, size_t len) { struct ionic_tx_stats *stats = q_to_tx_stats(q); - struct device *dev = q->lif->ionic->dev; + struct device *dev = q->dev; dma_addr_t dma_addr; dma_addr = skb_frag_dma_map(dev, frag, offset, len, DMA_TO_DEVICE); @@ -588,42 +605,72 @@ static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q, return dma_addr; } +static int ionic_tx_map_skb(struct ionic_queue *q, struct sk_buff *skb, + struct ionic_desc_info *desc_info) +{ + struct ionic_buf_info *buf_info = desc_info->bufs; + struct device *dev = q->dev; + dma_addr_t dma_addr; + unsigned int nfrags; + skb_frag_t *frag; + int frag_idx; + + dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb)); + if (dma_mapping_error(dev, dma_addr)) + return -EIO; + buf_info->dma_addr = dma_addr; + buf_info->len = skb_headlen(skb); + buf_info++; + + frag = skb_shinfo(skb)->frags; + nfrags = skb_shinfo(skb)->nr_frags; + for (frag_idx = 0; frag_idx < nfrags; frag_idx++, frag++) { + dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag)); + if (dma_mapping_error(dev, dma_addr)) + goto dma_fail; + buf_info->dma_addr = dma_addr; + buf_info->len = skb_frag_size(frag); + buf_info++; + } + + desc_info->nbufs = 1 + nfrags; + + return 0; + +dma_fail: + /* unwind the frag mappings and the head mapping */ + while (frag_idx > 0) { + frag_idx--; + buf_info--; + dma_unmap_page(dev, buf_info->dma_addr, + buf_info->len, DMA_TO_DEVICE); + } + dma_unmap_single(dev, buf_info->dma_addr, buf_info->len, DMA_TO_DEVICE); + return -EIO; +} + static void ionic_tx_clean(struct ionic_queue *q, struct ionic_desc_info *desc_info, struct ionic_cq_info *cq_info, void *cb_arg) { - struct ionic_txq_sg_desc *sg_desc = desc_info->sg_desc; - struct ionic_txq_sg_elem *elem = sg_desc->elems; + struct ionic_buf_info *buf_info = desc_info->bufs; struct ionic_tx_stats *stats = q_to_tx_stats(q); - struct ionic_txq_desc *desc = desc_info->desc; - struct device *dev = q->lif->ionic->dev; - u8 opcode, flags, nsge; + struct device *dev = q->dev; u16 queue_index; unsigned int i; - u64 addr; - - decode_txq_desc_cmd(le64_to_cpu(desc->cmd), - &opcode, &flags, &nsge, &addr); - - /* use unmap_single only if either this is not TSO, - * or this is first descriptor of a TSO - */ - if (opcode != IONIC_TXQ_DESC_OPCODE_TSO || - flags & IONIC_TXQ_DESC_FLAG_TSO_SOT) - dma_unmap_single(dev, (dma_addr_t)addr, - le16_to_cpu(desc->len), DMA_TO_DEVICE); - else - dma_unmap_page(dev, (dma_addr_t)addr, - le16_to_cpu(desc->len), DMA_TO_DEVICE); - for (i = 0; i < nsge; i++, elem++) - dma_unmap_page(dev, (dma_addr_t)le64_to_cpu(elem->addr), - le16_to_cpu(elem->len), DMA_TO_DEVICE); + if (desc_info->nbufs) { + dma_unmap_single(dev, (dma_addr_t)buf_info->dma_addr, + buf_info->len, DMA_TO_DEVICE); + buf_info++; + for (i = 1; i < desc_info->nbufs; i++, buf_info++) + dma_unmap_page(dev, (dma_addr_t)buf_info->dma_addr, + buf_info->len, DMA_TO_DEVICE); + } if (cb_arg) { struct sk_buff *skb = cb_arg; - u32 len = skb->len; queue_index = skb_get_queue_mapping(skb); if (unlikely(__netif_subqueue_stopped(q->lif->netdev, @@ -631,17 +678,21 @@ static void ionic_tx_clean(struct ionic_queue *q, netif_wake_subqueue(q->lif->netdev, queue_index); q->wake++; } - dev_kfree_skb_any(skb); + + desc_info->bytes = skb->len; stats->clean++; - netdev_tx_completed_queue(q_to_ndq(q), 1, len); + + dev_consume_skb_any(skb); } } static bool ionic_tx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info) { - struct ionic_txq_comp *comp = cq_info->cq_desc; + struct ionic_txq_comp *comp = cq_info->txcq; struct ionic_queue *q = cq->bound_q; struct ionic_desc_info *desc_info; + int bytes = 0; + int pkts = 0; u16 index; if (!color_match(comp->color, cq->done_color)) @@ -652,13 +703,21 @@ static bool ionic_tx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info) */ do { desc_info = &q->info[q->tail_idx]; + desc_info->bytes = 0; index = q->tail_idx; q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); ionic_tx_clean(q, desc_info, cq_info, desc_info->cb_arg); + if (desc_info->cb_arg) { + pkts++; + bytes += desc_info->bytes; + } desc_info->cb = NULL; desc_info->cb_arg = NULL; } while (index != le16_to_cpu(comp->comp_index)); + if (pkts && bytes) + netdev_tx_completed_queue(q_to_ndq(q), pkts, bytes); + return true; } @@ -677,15 +736,25 @@ void ionic_tx_flush(struct ionic_cq *cq) void ionic_tx_empty(struct ionic_queue *q) { struct ionic_desc_info *desc_info; + int bytes = 0; + int pkts = 0; /* walk the not completed tx entries, if any */ while (q->head_idx != q->tail_idx) { desc_info = &q->info[q->tail_idx]; + desc_info->bytes = 0; q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); ionic_tx_clean(q, desc_info, NULL, desc_info->cb_arg); + if (desc_info->cb_arg) { + pkts++; + bytes += desc_info->bytes; + } desc_info->cb = NULL; desc_info->cb_arg = NULL; } + + if (pkts && bytes) + netdev_tx_completed_queue(q_to_ndq(q), pkts, bytes); } static int ionic_tx_tcp_inner_pseudo_csum(struct sk_buff *skb) @@ -756,50 +825,33 @@ static void ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc desc->hdr_len = cpu_to_le16(hdrlen); desc->mss = cpu_to_le16(mss); - if (done) { + if (start) { skb_tx_timestamp(skb); netdev_tx_sent_queue(q_to_ndq(q), skb->len); - ionic_txq_post(q, !netdev_xmit_more(), ionic_tx_clean, skb); + ionic_txq_post(q, false, ionic_tx_clean, skb); } else { - ionic_txq_post(q, false, ionic_tx_clean, NULL); + ionic_txq_post(q, done, NULL, NULL); } } -static struct ionic_txq_desc *ionic_tx_tso_next(struct ionic_queue *q, - struct ionic_txq_sg_elem **elem) -{ - struct ionic_txq_sg_desc *sg_desc = q->info[q->head_idx].txq_sg_desc; - struct ionic_txq_desc *desc = q->info[q->head_idx].txq_desc; - - *elem = sg_desc->elems; - return desc; -} - static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb) { struct ionic_tx_stats *stats = q_to_tx_stats(q); - struct ionic_desc_info *rewind_desc_info; - struct device *dev = q->lif->ionic->dev; + struct ionic_desc_info *desc_info; + struct ionic_buf_info *buf_info; struct ionic_txq_sg_elem *elem; struct ionic_txq_desc *desc; - unsigned int frag_left = 0; - unsigned int offset = 0; - u16 abort = q->head_idx; - unsigned int len_left; + unsigned int chunk_len; + unsigned int frag_rem; + unsigned int tso_rem; + unsigned int seg_rem; dma_addr_t desc_addr; + dma_addr_t frag_addr; unsigned int hdrlen; - unsigned int nfrags; - unsigned int seglen; - u64 total_bytes = 0; - u64 total_pkts = 0; - u16 rewind = abort; - unsigned int left; unsigned int len; unsigned int mss; - skb_frag_t *frag; bool start, done; bool outer_csum; - dma_addr_t addr; bool has_vlan; u16 desc_len; u8 desc_nsge; @@ -807,9 +859,14 @@ static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb) bool encap; int err; + desc_info = &q->info[q->head_idx]; + buf_info = desc_info->bufs; + + if (unlikely(ionic_tx_map_skb(q, skb, desc_info))) + return -EIO; + + len = skb->len; mss = skb_shinfo(skb)->gso_size; - nfrags = skb_shinfo(skb)->nr_frags; - len_left = skb->len - skb_headlen(skb); outer_csum = (skb_shinfo(skb)->gso_type & SKB_GSO_GRE_CSUM) || (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM); has_vlan = !!skb_vlan_tag_present(skb); @@ -834,125 +891,75 @@ static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb) else hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb); - seglen = hdrlen + mss; - left = skb_headlen(skb); + tso_rem = len; + seg_rem = min(tso_rem, hdrlen + mss); - desc = ionic_tx_tso_next(q, &elem); - start = true; + frag_addr = 0; + frag_rem = 0; - /* Chop skb->data up into desc segments */ + start = true; - while (left > 0) { - len = min(seglen, left); - frag_left = seglen - len; - desc_addr = ionic_tx_map_single(q, skb->data + offset, len); - if (dma_mapping_error(dev, desc_addr)) - goto err_out_abort; - desc_len = len; + while (tso_rem > 0) { + desc = NULL; + elem = NULL; + desc_addr = 0; + desc_len = 0; desc_nsge = 0; - left -= len; - offset += len; - if (nfrags > 0 && frag_left > 0) - continue; - done = (nfrags == 0 && left == 0); - ionic_tx_tso_post(q, desc, skb, - desc_addr, desc_nsge, desc_len, - hdrlen, mss, - outer_csum, - vlan_tci, has_vlan, - start, done); - total_pkts++; - total_bytes += start ? len : len + hdrlen; - desc = ionic_tx_tso_next(q, &elem); - start = false; - seglen = mss; - } - - /* Chop skb frags into desc segments */ - - for (frag = skb_shinfo(skb)->frags; len_left; frag++) { - offset = 0; - left = skb_frag_size(frag); - len_left -= left; - nfrags--; - stats->frags++; - - while (left > 0) { - if (frag_left > 0) { - len = min(frag_left, left); - frag_left -= len; - addr = ionic_tx_map_frag(q, frag, offset, len); - if (dma_mapping_error(dev, addr)) - goto err_out_abort; - elem->addr = cpu_to_le64(addr); - elem->len = cpu_to_le16(len); + /* use fragments until we have enough to post a single descriptor */ + while (seg_rem > 0) { + /* if the fragment is exhausted then move to the next one */ + if (frag_rem == 0) { + /* grab the next fragment */ + frag_addr = buf_info->dma_addr; + frag_rem = buf_info->len; + buf_info++; + } + chunk_len = min(frag_rem, seg_rem); + if (!desc) { + /* fill main descriptor */ + desc = desc_info->txq_desc; + elem = desc_info->txq_sg_desc->elems; + desc_addr = frag_addr; + desc_len = chunk_len; + } else { + /* fill sg descriptor */ + elem->addr = cpu_to_le64(frag_addr); + elem->len = cpu_to_le16(chunk_len); elem++; desc_nsge++; - left -= len; - offset += len; - if (nfrags > 0 && frag_left > 0) - continue; - done = (nfrags == 0 && left == 0); - ionic_tx_tso_post(q, desc, skb, desc_addr, - desc_nsge, desc_len, - hdrlen, mss, outer_csum, - vlan_tci, has_vlan, - start, done); - total_pkts++; - total_bytes += start ? len : len + hdrlen; - desc = ionic_tx_tso_next(q, &elem); - start = false; - } else { - len = min(mss, left); - frag_left = mss - len; - desc_addr = ionic_tx_map_frag(q, frag, - offset, len); - if (dma_mapping_error(dev, desc_addr)) - goto err_out_abort; - desc_len = len; - desc_nsge = 0; - left -= len; - offset += len; - if (nfrags > 0 && frag_left > 0) - continue; - done = (nfrags == 0 && left == 0); - ionic_tx_tso_post(q, desc, skb, desc_addr, - desc_nsge, desc_len, - hdrlen, mss, outer_csum, - vlan_tci, has_vlan, - start, done); - total_pkts++; - total_bytes += start ? len : len + hdrlen; - desc = ionic_tx_tso_next(q, &elem); - start = false; } + frag_addr += chunk_len; + frag_rem -= chunk_len; + tso_rem -= chunk_len; + seg_rem -= chunk_len; } + seg_rem = min(tso_rem, mss); + done = (tso_rem == 0); + /* post descriptor */ + ionic_tx_tso_post(q, desc, skb, + desc_addr, desc_nsge, desc_len, + hdrlen, mss, outer_csum, vlan_tci, has_vlan, + start, done); + start = false; + /* Buffer information is stored with the first tso descriptor */ + desc_info = &q->info[q->head_idx]; + desc_info->nbufs = 0; } - stats->pkts += total_pkts; - stats->bytes += total_bytes; + stats->pkts += DIV_ROUND_UP(len - hdrlen, mss); + stats->bytes += len; stats->tso++; - stats->tso_bytes += total_bytes; + stats->tso_bytes = len; return 0; - -err_out_abort: - while (rewind != q->head_idx) { - rewind_desc_info = &q->info[rewind]; - ionic_tx_clean(q, rewind_desc_info, NULL, NULL); - rewind = (rewind + 1) & (q->num_descs - 1); - } - q->head_idx = abort; - - return -ENOMEM; } -static int ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb) +static int ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb, + struct ionic_desc_info *desc_info) { - struct ionic_txq_desc *desc = q->info[q->head_idx].txq_desc; + struct ionic_txq_desc *desc = desc_info->txq_desc; + struct ionic_buf_info *buf_info = desc_info->bufs; struct ionic_tx_stats *stats = q_to_tx_stats(q); - struct device *dev = q->lif->ionic->dev; - dma_addr_t dma_addr; bool has_vlan; u8 flags = 0; bool encap; @@ -961,23 +968,22 @@ static int ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb) has_vlan = !!skb_vlan_tag_present(skb); encap = skb->encapsulation; - dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb)); - if (dma_mapping_error(dev, dma_addr)) - return -ENOMEM; - flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_PARTIAL, - flags, skb_shinfo(skb)->nr_frags, dma_addr); + flags, skb_shinfo(skb)->nr_frags, + buf_info->dma_addr); desc->cmd = cpu_to_le64(cmd); - desc->len = cpu_to_le16(skb_headlen(skb)); - desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb)); - desc->csum_offset = cpu_to_le16(skb->csum_offset); + desc->len = cpu_to_le16(buf_info->len); if (has_vlan) { desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb)); stats->vlan_inserted++; + } else { + desc->vlan_tci = 0; } + desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb)); + desc->csum_offset = cpu_to_le16(skb->csum_offset); if (skb_csum_is_sctp(skb)) stats->crc32_csum++; @@ -987,12 +993,12 @@ static int ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb) return 0; } -static int ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb) +static int ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb, + struct ionic_desc_info *desc_info) { - struct ionic_txq_desc *desc = q->info[q->head_idx].txq_desc; + struct ionic_txq_desc *desc = desc_info->txq_desc; + struct ionic_buf_info *buf_info = desc_info->bufs; struct ionic_tx_stats *stats = q_to_tx_stats(q); - struct device *dev = q->lif->ionic->dev; - dma_addr_t dma_addr; bool has_vlan; u8 flags = 0; bool encap; @@ -1001,67 +1007,66 @@ static int ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb) has_vlan = !!skb_vlan_tag_present(skb); encap = skb->encapsulation; - dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb)); - if (dma_mapping_error(dev, dma_addr)) - return -ENOMEM; - flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE, - flags, skb_shinfo(skb)->nr_frags, dma_addr); + flags, skb_shinfo(skb)->nr_frags, + buf_info->dma_addr); desc->cmd = cpu_to_le64(cmd); - desc->len = cpu_to_le16(skb_headlen(skb)); + desc->len = cpu_to_le16(buf_info->len); if (has_vlan) { desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb)); stats->vlan_inserted++; + } else { + desc->vlan_tci = 0; } + desc->csum_start = 0; + desc->csum_offset = 0; stats->csum_none++; return 0; } -static int ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb) +static int ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb, + struct ionic_desc_info *desc_info) { - struct ionic_txq_sg_desc *sg_desc = q->info[q->head_idx].txq_sg_desc; - unsigned int len_left = skb->len - skb_headlen(skb); + struct ionic_txq_sg_desc *sg_desc = desc_info->txq_sg_desc; + struct ionic_buf_info *buf_info = &desc_info->bufs[1]; struct ionic_txq_sg_elem *elem = sg_desc->elems; struct ionic_tx_stats *stats = q_to_tx_stats(q); - struct device *dev = q->lif->ionic->dev; - dma_addr_t dma_addr; - skb_frag_t *frag; - u16 len; + unsigned int i; - for (frag = skb_shinfo(skb)->frags; len_left; frag++, elem++) { - len = skb_frag_size(frag); - elem->len = cpu_to_le16(len); - dma_addr = ionic_tx_map_frag(q, frag, 0, len); - if (dma_mapping_error(dev, dma_addr)) - return -ENOMEM; - elem->addr = cpu_to_le64(dma_addr); - len_left -= len; - stats->frags++; + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++, buf_info++, elem++) { + elem->addr = cpu_to_le64(buf_info->dma_addr); + elem->len = cpu_to_le16(buf_info->len); } + stats->frags += skb_shinfo(skb)->nr_frags; + return 0; } static int ionic_tx(struct ionic_queue *q, struct sk_buff *skb) { + struct ionic_desc_info *desc_info = &q->info[q->head_idx]; struct ionic_tx_stats *stats = q_to_tx_stats(q); int err; + if (unlikely(ionic_tx_map_skb(q, skb, desc_info))) + return -EIO; + /* set up the initial descriptor */ if (skb->ip_summed == CHECKSUM_PARTIAL) - err = ionic_tx_calc_csum(q, skb); + err = ionic_tx_calc_csum(q, skb, desc_info); else - err = ionic_tx_calc_no_csum(q, skb); + err = ionic_tx_calc_no_csum(q, skb, desc_info); if (err) return err; /* add frags */ - err = ionic_tx_skb_frags(q, skb); + err = ionic_tx_skb_frags(q, skb, desc_info); if (err) return err; @@ -1077,17 +1082,19 @@ static int ionic_tx(struct ionic_queue *q, struct sk_buff *skb) static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb) { - int sg_elems = q->lif->qtype_info[IONIC_QTYPE_TXQ].max_sg_elems; struct ionic_tx_stats *stats = q_to_tx_stats(q); + int ndescs; int err; - /* If TSO, need roundup(skb->len/mss) descs */ + /* Each desc is mss long max, so a descriptor for each gso_seg */ if (skb_is_gso(skb)) - return (skb->len / skb_shinfo(skb)->gso_size) + 1; + ndescs = skb_shinfo(skb)->gso_segs; + else + ndescs = 1; /* If non-TSO, just need 1 desc and nr_frags sg elems */ - if (skb_shinfo(skb)->nr_frags <= sg_elems) - return 1; + if (skb_shinfo(skb)->nr_frags <= q->max_sg_elems) + return ndescs; /* Too many frags, so linearize */ err = skb_linearize(skb); @@ -1096,8 +1103,7 @@ static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb) stats->linearize++; - /* Need 1 desc and zero sg elems */ - return 1; + return ndescs; } static int ionic_maybe_stop_tx(struct ionic_queue *q, int ndescs) |