diff options
Diffstat (limited to 'drivers/net/ethernet/cavium/liquidio/lio_main.c')
| -rw-r--r-- | drivers/net/ethernet/cavium/liquidio/lio_main.c | 956 |
1 files changed, 179 insertions, 777 deletions
diff --git a/drivers/net/ethernet/cavium/liquidio/lio_main.c b/drivers/net/ethernet/cavium/liquidio/lio_main.c index 120b6e537b28..e7f54948173f 100644 --- a/drivers/net/ethernet/cavium/liquidio/lio_main.c +++ b/drivers/net/ethernet/cavium/liquidio/lio_main.c @@ -39,10 +39,14 @@ MODULE_AUTHOR("Cavium Networks, <support@cavium.com>"); MODULE_DESCRIPTION("Cavium LiquidIO Intelligent Server Adapter Driver"); MODULE_LICENSE("GPL"); MODULE_VERSION(LIQUIDIO_VERSION); -MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210SV_NAME LIO_FW_NAME_SUFFIX); -MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210NV_NAME LIO_FW_NAME_SUFFIX); -MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_410NV_NAME LIO_FW_NAME_SUFFIX); -MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_23XX_NAME LIO_FW_NAME_SUFFIX); +MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210SV_NAME + "_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX); +MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210NV_NAME + "_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX); +MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_410NV_NAME + "_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX); +MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_23XX_NAME + "_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX); static int ddr_timeout = 10000; module_param(ddr_timeout, int, 0644); @@ -55,11 +59,24 @@ static int debug = -1; module_param(debug, int, 0644); MODULE_PARM_DESC(debug, "NETIF_MSG debug bits"); -static char fw_type[LIO_MAX_FW_TYPE_LEN]; -module_param_string(fw_type, fw_type, sizeof(fw_type), 0000); -MODULE_PARM_DESC(fw_type, "Type of firmware to be loaded. Default \"nic\""); +static char fw_type[LIO_MAX_FW_TYPE_LEN] = LIO_FW_NAME_TYPE_NIC; +module_param_string(fw_type, fw_type, sizeof(fw_type), 0444); +MODULE_PARM_DESC(fw_type, "Type of firmware to be loaded. Default \"nic\". Use \"none\" to load firmware from flash."); -static int ptp_enable = 1; +static u32 console_bitmask; +module_param(console_bitmask, int, 0644); +MODULE_PARM_DESC(console_bitmask, + "Bitmask indicating which consoles have debug output redirected to syslog."); + +/** + * \brief determines if a given console has debug enabled. + * @param console console to check + * @returns 1 = enabled. 0 otherwise + */ +static int octeon_console_debug_enabled(u32 console) +{ + return (console_bitmask >> (console)) & 0x1; +} /* Polling interval for determining when NIC application is alive */ #define LIQUIDIO_STARTER_POLL_INTERVAL_MS 100 @@ -158,16 +175,13 @@ struct handshake { int started_ok; }; -struct octeon_device_priv { - /** Tasklet structures for this device. */ - struct tasklet_struct droq_tasklet; - unsigned long napi_mask; -}; - #ifdef CONFIG_PCI_IOV static int liquidio_enable_sriov(struct pci_dev *dev, int num_vfs); #endif +static int octeon_dbg_console_print(struct octeon_device *oct, u32 console_num, + char *prefix, char *suffix); + static int octeon_device_init(struct octeon_device *); static int liquidio_stop(struct net_device *netdev); static void liquidio_remove(struct pci_dev *pdev); @@ -256,32 +270,6 @@ static void force_io_queues_off(struct octeon_device *oct) } /** - * \brief wait for all pending requests to complete - * @param oct Pointer to Octeon device - * - * Called during shutdown sequence - */ -static int wait_for_pending_requests(struct octeon_device *oct) -{ - int i, pcount = 0; - - for (i = 0; i < 100; i++) { - pcount = - atomic_read(&oct->response_list - [OCTEON_ORDERED_SC_LIST].pending_req_count); - if (pcount) - schedule_timeout_uninterruptible(HZ / 10); - else - break; - } - - if (pcount) - return 1; - - return 0; -} - -/** * \brief Cause device to go quiet so it can be safely removed/reset/etc * @param oct Pointer to Octeon device */ @@ -572,7 +560,7 @@ static inline void txqs_wake(struct net_device *netdev) for (i = 0; i < netdev->num_tx_queues; i++) { int qno = lio->linfo.txpciq[i % - (lio->linfo.num_txpciq)].s.q_no; + lio->oct_dev->num_iqs].s.q_no; if (__netif_subqueue_stopped(netdev, i)) { INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, qno, @@ -652,7 +640,7 @@ static inline int check_txq_status(struct lio *lio) /* check each sub-queue state */ for (q = 0; q < numqs; q++) { iq = lio->linfo.txpciq[q % - (lio->linfo.num_txpciq)].s.q_no; + lio->oct_dev->num_iqs].s.q_no; if (octnet_iq_is_full(lio->oct_dev, iq)) continue; if (__netif_subqueue_stopped(lio->netdev, q)) { @@ -823,7 +811,8 @@ static void print_link_info(struct net_device *netdev) { struct lio *lio = GET_LIO(netdev); - if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED) { + if (!ifstate_check(lio, LIO_IFSTATE_RESETTING) && + ifstate_check(lio, LIO_IFSTATE_REGISTERED)) { struct oct_link_info *linfo = &lio->linfo; if (linfo->link.s.link_up) { @@ -912,295 +901,6 @@ static inline void update_link_status(struct net_device *netdev, } } -/* Runs in interrupt context. */ -static void update_txq_status(struct octeon_device *oct, int iq_num) -{ - struct net_device *netdev; - struct lio *lio; - struct octeon_instr_queue *iq = oct->instr_queue[iq_num]; - - netdev = oct->props[iq->ifidx].netdev; - - /* This is needed because the first IQ does not have - * a netdev associated with it. - */ - if (!netdev) - return; - - lio = GET_LIO(netdev); - if (netif_is_multiqueue(netdev)) { - if (__netif_subqueue_stopped(netdev, iq->q_index) && - lio->linfo.link.s.link_up && - (!octnet_iq_is_full(oct, iq_num))) { - INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq_num, - tx_restart, 1); - netif_wake_subqueue(netdev, iq->q_index); - } - } else if (netif_queue_stopped(netdev) && - lio->linfo.link.s.link_up && - (!octnet_iq_is_full(oct, lio->txq))) { - INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, - lio->txq, tx_restart, 1); - netif_wake_queue(netdev); - } -} - -static -int liquidio_schedule_msix_droq_pkt_handler(struct octeon_droq *droq, u64 ret) -{ - struct octeon_device *oct = droq->oct_dev; - struct octeon_device_priv *oct_priv = - (struct octeon_device_priv *)oct->priv; - - if (droq->ops.poll_mode) { - droq->ops.napi_fn(droq); - } else { - if (ret & MSIX_PO_INT) { - tasklet_schedule(&oct_priv->droq_tasklet); - return 1; - } - /* this will be flushed periodically by check iq db */ - if (ret & MSIX_PI_INT) - return 0; - } - return 0; -} - -/** - * \brief Droq packet processor sceduler - * @param oct octeon device - */ -static void liquidio_schedule_droq_pkt_handlers(struct octeon_device *oct) -{ - struct octeon_device_priv *oct_priv = - (struct octeon_device_priv *)oct->priv; - u64 oq_no; - struct octeon_droq *droq; - - if (oct->int_status & OCT_DEV_INTR_PKT_DATA) { - for (oq_no = 0; oq_no < MAX_OCTEON_OUTPUT_QUEUES(oct); - oq_no++) { - if (!(oct->droq_intr & BIT_ULL(oq_no))) - continue; - - droq = oct->droq[oq_no]; - - if (droq->ops.poll_mode) { - droq->ops.napi_fn(droq); - oct_priv->napi_mask |= (1 << oq_no); - } else { - tasklet_schedule(&oct_priv->droq_tasklet); - } - } - } -} - -static irqreturn_t -liquidio_msix_intr_handler(int irq __attribute__((unused)), void *dev) -{ - u64 ret; - struct octeon_ioq_vector *ioq_vector = (struct octeon_ioq_vector *)dev; - struct octeon_device *oct = ioq_vector->oct_dev; - struct octeon_droq *droq = oct->droq[ioq_vector->droq_index]; - - ret = oct->fn_list.msix_interrupt_handler(ioq_vector); - - if ((ret & MSIX_PO_INT) || (ret & MSIX_PI_INT)) - liquidio_schedule_msix_droq_pkt_handler(droq, ret); - - return IRQ_HANDLED; -} - -/** - * \brief Interrupt handler for octeon - * @param irq unused - * @param dev octeon device - */ -static -irqreturn_t liquidio_legacy_intr_handler(int irq __attribute__((unused)), - void *dev) -{ - struct octeon_device *oct = (struct octeon_device *)dev; - irqreturn_t ret; - - /* Disable our interrupts for the duration of ISR */ - oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR); - - ret = oct->fn_list.process_interrupt_regs(oct); - - if (ret == IRQ_HANDLED) - liquidio_schedule_droq_pkt_handlers(oct); - - /* Re-enable our interrupts */ - if (!(atomic_read(&oct->status) == OCT_DEV_IN_RESET)) - oct->fn_list.enable_interrupt(oct, OCTEON_ALL_INTR); - - return ret; -} - -/** - * \brief Setup interrupt for octeon device - * @param oct octeon device - * - * Enable interrupt in Octeon device as given in the PCI interrupt mask. - */ -static int octeon_setup_interrupt(struct octeon_device *oct) -{ - int irqret, err; - struct msix_entry *msix_entries; - int i; - int num_ioq_vectors; - int num_alloc_ioq_vectors; - char *queue_irq_names = NULL; - char *aux_irq_name = NULL; - - if (OCTEON_CN23XX_PF(oct) && oct->msix_on) { - oct->num_msix_irqs = oct->sriov_info.num_pf_rings; - /* one non ioq interrupt for handling sli_mac_pf_int_sum */ - oct->num_msix_irqs += 1; - - /* allocate storage for the names assigned to each irq */ - oct->irq_name_storage = - kcalloc((MAX_IOQ_INTERRUPTS_PER_PF + 1), INTRNAMSIZ, - GFP_KERNEL); - if (!oct->irq_name_storage) { - dev_err(&oct->pci_dev->dev, "Irq name storage alloc failed...\n"); - return -ENOMEM; - } - - queue_irq_names = oct->irq_name_storage; - aux_irq_name = &queue_irq_names - [IRQ_NAME_OFF(MAX_IOQ_INTERRUPTS_PER_PF)]; - - oct->msix_entries = kcalloc( - oct->num_msix_irqs, sizeof(struct msix_entry), GFP_KERNEL); - if (!oct->msix_entries) { - dev_err(&oct->pci_dev->dev, "Memory Alloc failed...\n"); - kfree(oct->irq_name_storage); - oct->irq_name_storage = NULL; - return -ENOMEM; - } - - msix_entries = (struct msix_entry *)oct->msix_entries; - /*Assumption is that pf msix vectors start from pf srn to pf to - * trs and not from 0. if not change this code - */ - for (i = 0; i < oct->num_msix_irqs - 1; i++) - msix_entries[i].entry = oct->sriov_info.pf_srn + i; - msix_entries[oct->num_msix_irqs - 1].entry = - oct->sriov_info.trs; - num_alloc_ioq_vectors = pci_enable_msix_range( - oct->pci_dev, msix_entries, - oct->num_msix_irqs, - oct->num_msix_irqs); - if (num_alloc_ioq_vectors < 0) { - dev_err(&oct->pci_dev->dev, "unable to Allocate MSI-X interrupts\n"); - kfree(oct->msix_entries); - oct->msix_entries = NULL; - kfree(oct->irq_name_storage); - oct->irq_name_storage = NULL; - return num_alloc_ioq_vectors; - } - dev_dbg(&oct->pci_dev->dev, "OCTEON: Enough MSI-X interrupts are allocated...\n"); - - num_ioq_vectors = oct->num_msix_irqs; - - /** For PF, there is one non-ioq interrupt handler */ - num_ioq_vectors -= 1; - - snprintf(aux_irq_name, INTRNAMSIZ, - "LiquidIO%u-pf%u-aux", oct->octeon_id, oct->pf_num); - irqret = request_irq(msix_entries[num_ioq_vectors].vector, - liquidio_legacy_intr_handler, 0, - aux_irq_name, oct); - if (irqret) { - dev_err(&oct->pci_dev->dev, - "OCTEON: Request_irq failed for MSIX interrupt Error: %d\n", - irqret); - pci_disable_msix(oct->pci_dev); - kfree(oct->msix_entries); - oct->msix_entries = NULL; - kfree(oct->irq_name_storage); - oct->irq_name_storage = NULL; - return irqret; - } - - for (i = 0; i < num_ioq_vectors; i++) { - snprintf(&queue_irq_names[IRQ_NAME_OFF(i)], INTRNAMSIZ, - "LiquidIO%u-pf%u-rxtx-%u", - oct->octeon_id, oct->pf_num, i); - - irqret = request_irq(msix_entries[i].vector, - liquidio_msix_intr_handler, 0, - &queue_irq_names[IRQ_NAME_OFF(i)], - &oct->ioq_vector[i]); - if (irqret) { - dev_err(&oct->pci_dev->dev, - "OCTEON: Request_irq failed for MSIX interrupt Error: %d\n", - irqret); - /** Freeing the non-ioq irq vector here . */ - free_irq(msix_entries[num_ioq_vectors].vector, - oct); - - while (i) { - i--; - /** clearing affinity mask. */ - irq_set_affinity_hint( - msix_entries[i].vector, NULL); - free_irq(msix_entries[i].vector, - &oct->ioq_vector[i]); - } - pci_disable_msix(oct->pci_dev); - kfree(oct->msix_entries); - oct->msix_entries = NULL; - kfree(oct->irq_name_storage); - oct->irq_name_storage = NULL; - return irqret; - } - oct->ioq_vector[i].vector = msix_entries[i].vector; - /* assign the cpu mask for this msix interrupt vector */ - irq_set_affinity_hint( - msix_entries[i].vector, - (&oct->ioq_vector[i].affinity_mask)); - } - dev_dbg(&oct->pci_dev->dev, "OCTEON[%d]: MSI-X enabled\n", - oct->octeon_id); - } else { - err = pci_enable_msi(oct->pci_dev); - if (err) - dev_warn(&oct->pci_dev->dev, "Reverting to legacy interrupts. Error: %d\n", - err); - else - oct->flags |= LIO_FLAG_MSI_ENABLED; - - /* allocate storage for the names assigned to the irq */ - oct->irq_name_storage = kcalloc(1, INTRNAMSIZ, GFP_KERNEL); - if (!oct->irq_name_storage) - return -ENOMEM; - - queue_irq_names = oct->irq_name_storage; - - snprintf(&queue_irq_names[IRQ_NAME_OFF(0)], INTRNAMSIZ, - "LiquidIO%u-pf%u-rxtx-%u", - oct->octeon_id, oct->pf_num, 0); - - irqret = request_irq(oct->pci_dev->irq, - liquidio_legacy_intr_handler, - IRQF_SHARED, - &queue_irq_names[IRQ_NAME_OFF(0)], oct); - if (irqret) { - if (oct->flags & LIO_FLAG_MSI_ENABLED) - pci_disable_msi(oct->pci_dev); - dev_err(&oct->pci_dev->dev, "Request IRQ failed with code: %d\n", - irqret); - kfree(oct->irq_name_storage); - oct->irq_name_storage = NULL; - return irqret; - } - } - return 0; -} - static struct octeon_device *get_other_octeon_device(struct octeon_device *oct) { struct octeon_device *other_oct; @@ -1344,6 +1044,13 @@ liquidio_probe(struct pci_dev *pdev, if (pdev->device == OCTEON_CN23XX_PF_VID) oct_dev->msix_on = LIO_FLAG_MSIX_ENABLED; + /* Enable PTP for 6XXX Device */ + if (((pdev->device == OCTEON_CN66XX) || + (pdev->device == OCTEON_CN68XX))) + oct_dev->ptp_enable = true; + else + oct_dev->ptp_enable = false; + dev_info(&pdev->dev, "Initializing device %x:%x.\n", (u32)pdev->vendor, (u32)pdev->device); @@ -1415,6 +1122,33 @@ static bool fw_type_is_none(void) } /** + * \brief PCI FLR for each Octeon device. + * @param oct octeon device + */ +static void octeon_pci_flr(struct octeon_device *oct) +{ + int rc; + + pci_save_state(oct->pci_dev); + + pci_cfg_access_lock(oct->pci_dev); + + /* Quiesce the device completely */ + pci_write_config_word(oct->pci_dev, PCI_COMMAND, + PCI_COMMAND_INTX_DISABLE); + + rc = __pci_reset_function_locked(oct->pci_dev); + + if (rc != 0) + dev_err(&oct->pci_dev->dev, "Error %d resetting PCI function %d\n", + rc, oct->pf_num); + + pci_cfg_access_unlock(oct->pci_dev); + + pci_restore_state(oct->pci_dev); +} + +/** *\brief Destroy resources associated with octeon device * @param pdev PCI device structure * @param ent unused @@ -1474,11 +1208,15 @@ static void octeon_destroy_resources(struct octeon_device *oct) if (oct->msix_on) { msix_entries = (struct msix_entry *)oct->msix_entries; for (i = 0; i < oct->num_msix_irqs - 1; i++) { - /* clear the affinity_cpumask */ - irq_set_affinity_hint(msix_entries[i].vector, - NULL); - free_irq(msix_entries[i].vector, - &oct->ioq_vector[i]); + if (oct->ioq_vector[i].vector) { + /* clear the affinity_cpumask */ + irq_set_affinity_hint( + msix_entries[i].vector, + NULL); + free_irq(msix_entries[i].vector, + &oct->ioq_vector[i]); + oct->ioq_vector[i].vector = 0; + } } /* non-iov vector's argument is oct struct */ free_irq(msix_entries[i].vector, oct); @@ -1558,14 +1296,16 @@ static void octeon_destroy_resources(struct octeon_device *oct) case OCT_DEV_PCI_MAP_DONE: refcount = octeon_deregister_device(oct); - if (!fw_type_is_none()) { - /* Soft reset the octeon device before exiting. - * Implementation note: here, we reset the device - * if it is a CN6XXX OR the last CN23XX device. - */ - if (OCTEON_CN6XXX(oct) || !refcount) - oct->fn_list.soft_reset(oct); - } + /* Soft reset the octeon device before exiting. + * However, if fw was loaded from card (i.e. autoboot), + * perform an FLR instead. + * Implementation note: only soft-reset the device + * if it is a CN6XXX OR the LAST CN23XX device. + */ + if (fw_type_is_none()) + octeon_pci_flr(oct); + else if (OCTEON_CN6XXX(oct) || !refcount) + oct->fn_list.soft_reset(oct); octeon_unmap_pci_barx(oct, 0); octeon_unmap_pci_barx(oct, 1); @@ -1698,15 +1438,6 @@ static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx) if (atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) liquidio_stop(netdev); - if (fw_type_is_none()) { - struct octnic_ctrl_pkt nctrl; - - memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); - nctrl.ncmd.s.cmd = OCTNET_CMD_RESET_PF; - nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; - octnet_send_nic_ctrl_pkt(oct, &nctrl); - } - if (oct->props[lio->ifidx].napi_enabled == 1) { list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list) napi_disable(napi); @@ -1717,6 +1448,10 @@ static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx) oct->droq[0]->ops.poll_mode = 0; } + /* Delete NAPI */ + list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list) + netif_napi_del(napi); + if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED) unregister_netdev(netdev); @@ -1754,7 +1489,7 @@ static int liquidio_stop_nic_module(struct octeon_device *oct) for (i = 0; i < oct->ifcount; i++) { lio = GET_LIO(oct->props[i].netdev); - for (j = 0; j < lio->linfo.num_rxpciq; j++) + for (j = 0; j < oct->num_oqs; j++) octeon_unregister_droq_ops(oct, lio->linfo.rxpciq[j].s.q_no); } @@ -1825,6 +1560,13 @@ static int octeon_chip_specific_setup(struct octeon_device *oct) case OCTEON_CN23XX_PCIID_PF: oct->chip_id = OCTEON_CN23XX_PF_VID; ret = setup_cn23xx_octeon_pf_device(oct); + if (ret) + break; +#ifdef CONFIG_PCI_IOV + if (!ret) + pci_sriov_set_totalvfs(oct->pci_dev, + oct->sriov_info.max_vfs); +#endif s = "CN23XX"; break; @@ -1889,7 +1631,7 @@ static inline int check_txq_state(struct lio *lio, struct sk_buff *skb) if (netif_is_multiqueue(lio->netdev)) { q = skb->queue_mapping; - iq = lio->linfo.txpciq[(q % (lio->linfo.num_txpciq))].s.q_no; + iq = lio->linfo.txpciq[(q % lio->oct_dev->num_iqs)].s.q_no; } else { iq = lio->txq; q = iq; @@ -2192,11 +1934,6 @@ static int load_firmware(struct octeon_device *oct) char fw_name[LIO_MAX_FW_FILENAME_LEN]; char *tmp_fw_type; - if (fw_type_is_none()) { - dev_info(&oct->pci_dev->dev, "Skipping firmware load\n"); - return ret; - } - if (fw_type[0] == '\0') tmp_fw_type = LIO_FW_NAME_TYPE_NIC; else @@ -2222,43 +1959,6 @@ static int load_firmware(struct octeon_device *oct) } /** - * \brief Setup output queue - * @param oct octeon device - * @param q_no which queue - * @param num_descs how many descriptors - * @param desc_size size of each descriptor - * @param app_ctx application context - */ -static int octeon_setup_droq(struct octeon_device *oct, int q_no, int num_descs, - int desc_size, void *app_ctx) -{ - int ret_val = 0; - - dev_dbg(&oct->pci_dev->dev, "Creating Droq: %d\n", q_no); - /* droq creation and local register settings. */ - ret_val = octeon_create_droq(oct, q_no, num_descs, desc_size, app_ctx); - if (ret_val < 0) - return ret_val; - - if (ret_val == 1) { - dev_dbg(&oct->pci_dev->dev, "Using default droq %d\n", q_no); - return 0; - } - /* tasklet creation for the droq */ - - /* Enable the droq queues */ - octeon_set_droq_pkt_op(oct, q_no, 1); - - /* Send Credit for Octeon Output queues. Credits are always - * sent after the output queue is enabled. - */ - writel(oct->droq[q_no]->max_count, - oct->droq[q_no]->pkts_credit_reg); - - return ret_val; -} - -/** * \brief Callback for getting interface configuration * @param status status of request * @param buf pointer to resp structure @@ -2291,352 +1991,6 @@ static void if_cfg_callback(struct octeon_device *oct, wake_up_interruptible(&ctx->wc); } -/** Routine to push packets arriving on Octeon interface upto network layer. - * @param oct_id - octeon device id. - * @param skbuff - skbuff struct to be passed to network layer. - * @param len - size of total data received. - * @param rh - Control header associated with the packet - * @param param - additional control data with the packet - * @param arg - farg registered in droq_ops - */ -static void -liquidio_push_packet(u32 octeon_id __attribute__((unused)), - void *skbuff, - u32 len, - union octeon_rh *rh, - void *param, - void *arg) -{ - struct napi_struct *napi = param; - struct sk_buff *skb = (struct sk_buff *)skbuff; - struct skb_shared_hwtstamps *shhwtstamps; - u64 ns; - u16 vtag = 0; - u32 r_dh_off; - struct net_device *netdev = (struct net_device *)arg; - struct octeon_droq *droq = container_of(param, struct octeon_droq, - napi); - if (netdev) { - int packet_was_received; - struct lio *lio = GET_LIO(netdev); - struct octeon_device *oct = lio->oct_dev; - - /* Do not proceed if the interface is not in RUNNING state. */ - if (!ifstate_check(lio, LIO_IFSTATE_RUNNING)) { - recv_buffer_free(skb); - droq->stats.rx_dropped++; - return; - } - - skb->dev = netdev; - - skb_record_rx_queue(skb, droq->q_no); - if (likely(len > MIN_SKB_SIZE)) { - struct octeon_skb_page_info *pg_info; - unsigned char *va; - - pg_info = ((struct octeon_skb_page_info *)(skb->cb)); - if (pg_info->page) { - /* For Paged allocation use the frags */ - va = page_address(pg_info->page) + - pg_info->page_offset; - memcpy(skb->data, va, MIN_SKB_SIZE); - skb_put(skb, MIN_SKB_SIZE); - skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, - pg_info->page, - pg_info->page_offset + - MIN_SKB_SIZE, - len - MIN_SKB_SIZE, - LIO_RXBUFFER_SZ); - } - } else { - struct octeon_skb_page_info *pg_info = - ((struct octeon_skb_page_info *)(skb->cb)); - skb_copy_to_linear_data(skb, page_address(pg_info->page) - + pg_info->page_offset, len); - skb_put(skb, len); - put_page(pg_info->page); - } - - r_dh_off = (rh->r_dh.len - 1) * BYTES_PER_DHLEN_UNIT; - - if (((oct->chip_id == OCTEON_CN66XX) || - (oct->chip_id == OCTEON_CN68XX)) && - ptp_enable) { - if (rh->r_dh.has_hwtstamp) { - /* timestamp is included from the hardware at - * the beginning of the packet. - */ - if (ifstate_check - (lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED)) { - /* Nanoseconds are in the first 64-bits - * of the packet. - */ - memcpy(&ns, (skb->data + r_dh_off), - sizeof(ns)); - r_dh_off -= BYTES_PER_DHLEN_UNIT; - shhwtstamps = skb_hwtstamps(skb); - shhwtstamps->hwtstamp = - ns_to_ktime(ns + - lio->ptp_adjust); - } - } - } - - if (rh->r_dh.has_hash) { - __be32 *hash_be = (__be32 *)(skb->data + r_dh_off); - u32 hash = be32_to_cpu(*hash_be); - - skb_set_hash(skb, hash, PKT_HASH_TYPE_L4); - r_dh_off -= BYTES_PER_DHLEN_UNIT; - } - - skb_pull(skb, rh->r_dh.len * BYTES_PER_DHLEN_UNIT); - - skb->protocol = eth_type_trans(skb, skb->dev); - if ((netdev->features & NETIF_F_RXCSUM) && - (((rh->r_dh.encap_on) && - (rh->r_dh.csum_verified & CNNIC_TUN_CSUM_VERIFIED)) || - (!(rh->r_dh.encap_on) && - (rh->r_dh.csum_verified & CNNIC_CSUM_VERIFIED)))) - /* checksum has already been verified */ - skb->ip_summed = CHECKSUM_UNNECESSARY; - else - skb->ip_summed = CHECKSUM_NONE; - - /* Setting Encapsulation field on basis of status received - * from the firmware - */ - if (rh->r_dh.encap_on) { - skb->encapsulation = 1; - skb->csum_level = 1; - droq->stats.rx_vxlan++; - } - - /* inbound VLAN tag */ - if ((netdev->features & NETIF_F_HW_VLAN_CTAG_RX) && - (rh->r_dh.vlan != 0)) { - u16 vid = rh->r_dh.vlan; - u16 priority = rh->r_dh.priority; - - vtag = priority << 13 | vid; - __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vtag); - } - - packet_was_received = napi_gro_receive(napi, skb) != GRO_DROP; - - if (packet_was_received) { - droq->stats.rx_bytes_received += len; - droq->stats.rx_pkts_received++; - } else { - droq->stats.rx_dropped++; - netif_info(lio, rx_err, lio->netdev, - "droq:%d error rx_dropped:%llu\n", - droq->q_no, droq->stats.rx_dropped); - } - - } else { - recv_buffer_free(skb); - } -} - -/** - * \brief wrapper for calling napi_schedule - * @param param parameters to pass to napi_schedule - * - * Used when scheduling on different CPUs - */ -static void napi_schedule_wrapper(void *param) -{ - struct napi_struct *napi = param; - - napi_schedule(napi); -} - -/** - * \brief callback when receive interrupt occurs and we are in NAPI mode - * @param arg pointer to octeon output queue - */ -static void liquidio_napi_drv_callback(void *arg) -{ - struct octeon_device *oct; - struct octeon_droq *droq = arg; - int this_cpu = smp_processor_id(); - - oct = droq->oct_dev; - - if (OCTEON_CN23XX_PF(oct) || droq->cpu_id == this_cpu) { - napi_schedule_irqoff(&droq->napi); - } else { - call_single_data_t *csd = &droq->csd; - - csd->func = napi_schedule_wrapper; - csd->info = &droq->napi; - csd->flags = 0; - - smp_call_function_single_async(droq->cpu_id, csd); - } -} - -/** - * \brief Entry point for NAPI polling - * @param napi NAPI structure - * @param budget maximum number of items to process - */ -static int liquidio_napi_poll(struct napi_struct *napi, int budget) -{ - struct octeon_droq *droq; - int work_done; - int tx_done = 0, iq_no; - struct octeon_instr_queue *iq; - struct octeon_device *oct; - - droq = container_of(napi, struct octeon_droq, napi); - oct = droq->oct_dev; - iq_no = droq->q_no; - /* Handle Droq descriptors */ - work_done = octeon_process_droq_poll_cmd(oct, droq->q_no, - POLL_EVENT_PROCESS_PKTS, - budget); - - /* Flush the instruction queue */ - iq = oct->instr_queue[iq_no]; - if (iq) { - if (atomic_read(&iq->instr_pending)) - /* Process iq buffers with in the budget limits */ - tx_done = octeon_flush_iq(oct, iq, budget); - else - tx_done = 1; - /* Update iq read-index rather than waiting for next interrupt. - * Return back if tx_done is false. - */ - update_txq_status(oct, iq_no); - } else { - dev_err(&oct->pci_dev->dev, "%s: iq (%d) num invalid\n", - __func__, iq_no); - } - - /* force enable interrupt if reg cnts are high to avoid wraparound */ - if ((work_done < budget && tx_done) || - (iq && iq->pkt_in_done >= MAX_REG_CNT) || - (droq->pkt_count >= MAX_REG_CNT)) { - tx_done = 1; - napi_complete_done(napi, work_done); - octeon_process_droq_poll_cmd(droq->oct_dev, droq->q_no, - POLL_EVENT_ENABLE_INTR, 0); - return 0; - } - - return (!tx_done) ? (budget) : (work_done); -} - -/** - * \brief Setup input and output queues - * @param octeon_dev octeon device - * @param ifidx Interface Index - * - * Note: Queues are with respect to the octeon device. Thus - * an input queue is for egress packets, and output queues - * are for ingress packets. - */ -static inline int setup_io_queues(struct octeon_device *octeon_dev, - int ifidx) -{ - struct octeon_droq_ops droq_ops; - struct net_device *netdev; - static int cpu_id; - static int cpu_id_modulus; - struct octeon_droq *droq; - struct napi_struct *napi; - int q, q_no, retval = 0; - struct lio *lio; - int num_tx_descs; - - netdev = octeon_dev->props[ifidx].netdev; - - lio = GET_LIO(netdev); - - memset(&droq_ops, 0, sizeof(struct octeon_droq_ops)); - - droq_ops.fptr = liquidio_push_packet; - droq_ops.farg = (void *)netdev; - - droq_ops.poll_mode = 1; - droq_ops.napi_fn = liquidio_napi_drv_callback; - cpu_id = 0; - cpu_id_modulus = num_present_cpus(); - - /* set up DROQs. */ - for (q = 0; q < lio->linfo.num_rxpciq; q++) { - q_no = lio->linfo.rxpciq[q].s.q_no; - dev_dbg(&octeon_dev->pci_dev->dev, - "setup_io_queues index:%d linfo.rxpciq.s.q_no:%d\n", - q, q_no); - retval = octeon_setup_droq(octeon_dev, q_no, - CFG_GET_NUM_RX_DESCS_NIC_IF - (octeon_get_conf(octeon_dev), - lio->ifidx), - CFG_GET_NUM_RX_BUF_SIZE_NIC_IF - (octeon_get_conf(octeon_dev), - lio->ifidx), NULL); - if (retval) { - dev_err(&octeon_dev->pci_dev->dev, - "%s : Runtime DROQ(RxQ) creation failed.\n", - __func__); - return 1; - } - - droq = octeon_dev->droq[q_no]; - napi = &droq->napi; - dev_dbg(&octeon_dev->pci_dev->dev, "netif_napi_add netdev:%llx oct:%llx pf_num:%d\n", - (u64)netdev, (u64)octeon_dev, octeon_dev->pf_num); - netif_napi_add(netdev, napi, liquidio_napi_poll, 64); - - /* designate a CPU for this droq */ - droq->cpu_id = cpu_id; - cpu_id++; - if (cpu_id >= cpu_id_modulus) - cpu_id = 0; - - octeon_register_droq_ops(octeon_dev, q_no, &droq_ops); - } - - if (OCTEON_CN23XX_PF(octeon_dev)) { - /* 23XX PF can receive control messages (via the first PF-owned - * droq) from the firmware even if the ethX interface is down, - * so that's why poll_mode must be off for the first droq. - */ - octeon_dev->droq[0]->ops.poll_mode = 0; - } - - /* set up IQs. */ - for (q = 0; q < lio->linfo.num_txpciq; q++) { - num_tx_descs = CFG_GET_NUM_TX_DESCS_NIC_IF(octeon_get_conf - (octeon_dev), - lio->ifidx); - retval = octeon_setup_iq(octeon_dev, ifidx, q, - lio->linfo.txpciq[q], num_tx_descs, - netdev_get_tx_queue(netdev, q)); - if (retval) { - dev_err(&octeon_dev->pci_dev->dev, - " %s : Runtime IQ(TxQ) creation failed.\n", - __func__); - return 1; - } - - if (octeon_dev->ioq_vector) { - struct octeon_ioq_vector *ioq_vector; - - ioq_vector = &octeon_dev->ioq_vector[q]; - netif_set_xps_queue(netdev, - &ioq_vector->affinity_mask, - ioq_vector->iq_index); - } - } - - return 0; -} - /** * \brief Poll routine for checking transmit queue status * @param work work_struct data structure @@ -2707,8 +2061,7 @@ static int liquidio_open(struct net_device *netdev) oct->droq[0]->ops.poll_mode = 1; } - if ((oct->chip_id == OCTEON_CN66XX || oct->chip_id == OCTEON_CN68XX) && - ptp_enable) + if (oct->ptp_enable) oct_ptp_open(netdev); ifstate_set(lio, LIO_IFSTATE_RUNNING); @@ -2746,6 +2099,17 @@ static int liquidio_stop(struct net_device *netdev) { struct lio *lio = GET_LIO(netdev); struct octeon_device *oct = lio->oct_dev; + struct napi_struct *napi, *n; + + if (oct->props[lio->ifidx].napi_enabled) { + list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list) + napi_disable(napi); + + oct->props[lio->ifidx].napi_enabled = 0; + + if (OCTEON_CN23XX_PF(oct)) + oct->droq[0]->ops.poll_mode = 0; + } ifstate_reset(lio, LIO_IFSTATE_RUNNING); @@ -2916,7 +2280,10 @@ static struct net_device_stats *liquidio_get_stats(struct net_device *netdev) oct = lio->oct_dev; - for (i = 0; i < lio->linfo.num_txpciq; i++) { + if (ifstate_check(lio, LIO_IFSTATE_RESETTING)) + return stats; + + for (i = 0; i < oct->num_iqs; i++) { iq_no = lio->linfo.txpciq[i].s.q_no; iq_stats = &oct->instr_queue[iq_no]->stats; pkts += iq_stats->tx_done; @@ -2932,7 +2299,7 @@ static struct net_device_stats *liquidio_get_stats(struct net_device *netdev) drop = 0; bytes = 0; - for (i = 0; i < lio->linfo.num_rxpciq; i++) { + for (i = 0; i < oct->num_oqs; i++) { oq_no = lio->linfo.rxpciq[i].s.q_no; oq_stats = &oct->droq[oq_no]->stats; pkts += oq_stats->rx_pkts_received; @@ -3052,8 +2419,7 @@ static int liquidio_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) switch (cmd) { case SIOCSHWTSTAMP: - if ((lio->oct_dev->chip_id == OCTEON_CN66XX || - lio->oct_dev->chip_id == OCTEON_CN68XX) && ptp_enable) + if (lio->oct_dev->ptp_enable) return hwtstamp_ioctl(netdev, ifr); default: return -EOPNOTSUPP; @@ -4188,7 +3554,9 @@ static int setup_nic_devices(struct octeon_device *octeon_dev) */ lio->txq = lio->linfo.txpciq[0].s.q_no; lio->rxq = lio->linfo.rxpciq[0].s.q_no; - if (setup_io_queues(octeon_dev, i)) { + if (liquidio_setup_io_queues(octeon_dev, i, + lio->linfo.num_txpciq, + lio->linfo.num_rxpciq)) { dev_err(&octeon_dev->pci_dev->dev, "I/O queues creation failed\n"); goto setup_nic_dev_fail; } @@ -4516,6 +3884,7 @@ static int octeon_device_init(struct octeon_device *octeon_dev) int j, ret; int fw_loaded = 0; char bootcmd[] = "\n"; + char *dbg_enb = NULL; struct octeon_device_priv *oct_priv = (struct octeon_device_priv *)octeon_dev->priv; atomic_set(&octeon_dev->status, OCT_DEV_BEGIN_STATE); @@ -4548,18 +3917,16 @@ static int octeon_device_init(struct octeon_device *octeon_dev) octeon_dev->app_mode = CVM_DRV_INVALID_APP; if (OCTEON_CN23XX_PF(octeon_dev)) { - if (!cn23xx_fw_loaded(octeon_dev)) { + if (!cn23xx_fw_loaded(octeon_dev) && !fw_type_is_none()) { fw_loaded = 0; - if (!fw_type_is_none()) { - /* Do a soft reset of the Octeon device. */ - if (octeon_dev->fn_list.soft_reset(octeon_dev)) - return 1; - /* things might have changed */ - if (!cn23xx_fw_loaded(octeon_dev)) - fw_loaded = 0; - else - fw_loaded = 1; - } + /* Do a soft reset of the Octeon device. */ + if (octeon_dev->fn_list.soft_reset(octeon_dev)) + return 1; + /* things might have changed */ + if (!cn23xx_fw_loaded(octeon_dev)) + fw_loaded = 0; + else + fw_loaded = 1; } else { fw_loaded = 1; } @@ -4666,7 +4033,8 @@ static int octeon_device_init(struct octeon_device *octeon_dev) /* Setup the interrupt handler and record the INT SUM register address */ - if (octeon_setup_interrupt(octeon_dev)) + if (octeon_setup_interrupt(octeon_dev, + octeon_dev->sriov_info.num_pf_rings)) return 1; /* Enable Octeon device interrupts */ @@ -4674,6 +4042,18 @@ static int octeon_device_init(struct octeon_device *octeon_dev) atomic_set(&octeon_dev->status, OCT_DEV_INTR_SET_DONE); + /* Send Credit for Octeon Output queues. Credits are always sent BEFORE + * the output queue is enabled. + * This ensures that we'll receive the f/w CORE DRV_ACTIVE message in + * case we've configured CN23XX_SLI_GBL_CONTROL[NOPTR_D] = 0. + * Otherwise, it is possible that the DRV_ACTIVE message will be sent + * before any credits have been issued, causing the ring to be reset + * (and the f/w appear to never have started). + */ + for (j = 0; j < octeon_dev->num_oqs; j++) + writel(octeon_dev->droq[j]->max_count, + octeon_dev->droq[j]->pkts_credit_reg); + /* Enable the input and output queues for this Octeon device */ ret = octeon_dev->fn_list.enable_io_queues(octeon_dev); if (ret) { @@ -4722,10 +4102,19 @@ static int octeon_device_init(struct octeon_device *octeon_dev) dev_err(&octeon_dev->pci_dev->dev, "Could not access board consoles\n"); return 1; } - ret = octeon_add_console(octeon_dev, 0); + /* If console debug enabled, specify empty string to use default + * enablement ELSE specify NULL string for 'disabled'. + */ + dbg_enb = octeon_console_debug_enabled(0) ? "" : NULL; + ret = octeon_add_console(octeon_dev, 0, dbg_enb); if (ret) { dev_err(&octeon_dev->pci_dev->dev, "Could not access board console\n"); return 1; + } else if (octeon_console_debug_enabled(0)) { + /* If console was added AND we're logging console output + * then set our console print function. + */ + octeon_dev->console[0].print = octeon_dbg_console_print; } atomic_set(&octeon_dev->status, OCT_DEV_CONSOLE_INIT_DONE); @@ -4736,12 +4125,6 @@ static int octeon_device_init(struct octeon_device *octeon_dev) dev_err(&octeon_dev->pci_dev->dev, "Could not load firmware to board\n"); return 1; } - /* set bit 1 of SLI_SCRATCH_1 to indicate that firmware is - * loaded - */ - if (OCTEON_CN23XX_PF(octeon_dev)) - octeon_write_csr64(octeon_dev, CN23XX_SLI_SCRATCH1, - 2ULL); } handshake[octeon_dev->octeon_id].init_ok = 1; @@ -4749,14 +4132,33 @@ static int octeon_device_init(struct octeon_device *octeon_dev) atomic_set(&octeon_dev->status, OCT_DEV_HOST_OK); - /* Send Credit for Octeon Output queues. Credits are always sent after - * the output queue is enabled. - */ - for (j = 0; j < octeon_dev->num_oqs; j++) - writel(octeon_dev->droq[j]->max_count, - octeon_dev->droq[j]->pkts_credit_reg); + return 0; +} + +/** + * \brief Debug console print function + * @param octeon_dev octeon device + * @param console_num console number + * @param prefix first portion of line to display + * @param suffix second portion of line to display + * + * The OCTEON debug console outputs entire lines (excluding '\n'). + * Normally, the line will be passed in the 'prefix' parameter. + * However, due to buffering, it is possible for a line to be split into two + * parts, in which case they will be passed as the 'prefix' parameter and + * 'suffix' parameter. + */ +static int octeon_dbg_console_print(struct octeon_device *oct, u32 console_num, + char *prefix, char *suffix) +{ + if (prefix && suffix) + dev_info(&oct->pci_dev->dev, "%u: %s%s\n", console_num, prefix, + suffix); + else if (prefix) + dev_info(&oct->pci_dev->dev, "%u: %s\n", console_num, prefix); + else if (suffix) + dev_info(&oct->pci_dev->dev, "%u: %s\n", console_num, suffix); - /* Packets can start arriving on the output queues from this point. */ return 0; } |