diff options
author | Arnaldo Carvalho de Melo <acme@redhat.com> | 2020-08-06 14:15:47 +0300 |
---|---|---|
committer | Arnaldo Carvalho de Melo <acme@redhat.com> | 2020-08-06 14:15:47 +0300 |
commit | 94fb1afb14c4f0ceb8c5508ddddac6819f662e95 (patch) | |
tree | 4988e5769dc7482caa7f441475ae31f50bbd37ef /drivers/net/ethernet/amazon/ena/ena_netdev.c | |
parent | c4735d990268399da9133b0ad445e488ece009ad (diff) | |
parent | 47ec5303d73ea344e84f46660fff693c57641386 (diff) | |
download | linux-94fb1afb14c4f0ceb8c5508ddddac6819f662e95.tar.xz |
Mgerge remote-tracking branch 'torvalds/master' into perf/core
To sync headers, for instance, in this case tools/perf was ahead of
upstream till Linus merged tip/perf/core to get the
PERF_RECORD_TEXT_POKE changes:
Warning: Kernel ABI header at 'tools/include/uapi/linux/perf_event.h' differs from latest version at 'include/uapi/linux/perf_event.h'
diff -u tools/include/uapi/linux/perf_event.h include/uapi/linux/perf_event.h
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
Diffstat (limited to 'drivers/net/ethernet/amazon/ena/ena_netdev.c')
-rw-r--r-- | drivers/net/ethernet/amazon/ena/ena_netdev.c | 211 |
1 files changed, 115 insertions, 96 deletions
diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c index dda4b8fc9525..2a6c9725e092 100644 --- a/drivers/net/ethernet/amazon/ena/ena_netdev.c +++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c @@ -307,7 +307,7 @@ static int ena_xdp_xmit_buff(struct net_device *dev, struct ena_rx_buffer *rx_info) { struct ena_adapter *adapter = netdev_priv(dev); - struct ena_com_tx_ctx ena_tx_ctx = {0}; + struct ena_com_tx_ctx ena_tx_ctx = {}; struct ena_tx_buffer *tx_info; struct ena_ring *xdp_ring; u16 next_to_use, req_id; @@ -576,15 +576,9 @@ static int ena_xdp_set(struct net_device *netdev, struct netdev_bpf *bpf) */ static int ena_xdp(struct net_device *netdev, struct netdev_bpf *bpf) { - struct ena_adapter *adapter = netdev_priv(netdev); - switch (bpf->command) { case XDP_SETUP_PROG: return ena_xdp_set(netdev, bpf); - case XDP_QUERY_PROG: - bpf->prog_id = adapter->xdp_bpf_prog ? - adapter->xdp_bpf_prog->aux->id : 0; - break; default: return -EINVAL; } @@ -655,6 +649,7 @@ static void ena_init_io_rings(struct ena_adapter *adapter, txr->sgl_size = adapter->max_tx_sgl_size; txr->smoothed_interval = ena_com_get_nonadaptive_moderation_interval_tx(ena_dev); + txr->disable_meta_caching = adapter->disable_meta_caching; /* Don't init RX queues for xdp queues */ if (!ENA_IS_XDP_INDEX(adapter, i)) { @@ -959,8 +954,11 @@ static int ena_alloc_rx_page(struct ena_ring *rx_ring, return -ENOMEM; } + /* To enable NIC-side port-mirroring, AKA SPAN port, + * we make the buffer readable from the nic as well + */ dma = dma_map_page(rx_ring->dev, page, 0, ENA_PAGE_SIZE, - DMA_FROM_DEVICE); + DMA_BIDIRECTIONAL); if (unlikely(dma_mapping_error(rx_ring->dev, dma))) { u64_stats_update_begin(&rx_ring->syncp); rx_ring->rx_stats.dma_mapping_err++; @@ -993,10 +991,9 @@ static void ena_free_rx_page(struct ena_ring *rx_ring, return; } - dma_unmap_page(rx_ring->dev, - ena_buf->paddr - rx_ring->rx_headroom, + dma_unmap_page(rx_ring->dev, ena_buf->paddr - rx_ring->rx_headroom, ENA_PAGE_SIZE, - DMA_FROM_DEVICE); + DMA_BIDIRECTIONAL); __free_page(page); rx_info->page = NULL; @@ -1431,7 +1428,7 @@ static struct sk_buff *ena_rx_skb(struct ena_ring *rx_ring, do { dma_unmap_page(rx_ring->dev, dma_unmap_addr(&rx_info->ena_buf, paddr), - ENA_PAGE_SIZE, DMA_FROM_DEVICE); + ENA_PAGE_SIZE, DMA_BIDIRECTIONAL); skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_info->page, rx_info->page_offset, len, ENA_PAGE_SIZE); @@ -1913,7 +1910,10 @@ static int ena_io_poll(struct napi_struct *napi, int budget) /* Update numa and unmask the interrupt only when schedule * from the interrupt context (vs from sk_busy_loop) */ - if (napi_complete_done(napi, rx_work_done)) { + if (napi_complete_done(napi, rx_work_done) && + READ_ONCE(ena_napi->interrupts_masked)) { + smp_rmb(); /* make sure interrupts_masked is read */ + WRITE_ONCE(ena_napi->interrupts_masked, false); /* We apply adaptive moderation on Rx path only. * Tx uses static interrupt moderation. */ @@ -1961,6 +1961,9 @@ static irqreturn_t ena_intr_msix_io(int irq, void *data) ena_napi->first_interrupt = true; + WRITE_ONCE(ena_napi->interrupts_masked, true); + smp_wmb(); /* write interrupts_masked before calling napi */ + napi_schedule_irqoff(&ena_napi->napi); return IRQ_HANDLED; @@ -2190,14 +2193,13 @@ static void ena_del_napi_in_range(struct ena_adapter *adapter, static void ena_init_napi_in_range(struct ena_adapter *adapter, int first_index, int count) { - struct ena_napi *napi = {0}; int i; for (i = first_index; i < first_index + count; i++) { - napi = &adapter->ena_napi[i]; + struct ena_napi *napi = &adapter->ena_napi[i]; netif_napi_add(adapter->netdev, - &adapter->ena_napi[i].napi, + &napi->napi, ENA_IS_XDP_INDEX(adapter, i) ? ena_xdp_io_poll : ena_io_poll, ENA_NAPI_BUDGET); @@ -2776,7 +2778,9 @@ int ena_update_queue_count(struct ena_adapter *adapter, u32 new_channel_count) return dev_was_up ? ena_open(adapter->netdev) : 0; } -static void ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx, struct sk_buff *skb) +static void ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx, + struct sk_buff *skb, + bool disable_meta_caching) { u32 mss = skb_shinfo(skb)->gso_size; struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta; @@ -2820,7 +2824,9 @@ static void ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx, struct sk_buff *skb) ena_meta->l3_hdr_len = skb_network_header_len(skb); ena_meta->l3_hdr_offset = skb_network_offset(skb); ena_tx_ctx->meta_valid = 1; - + } else if (disable_meta_caching) { + memset(ena_meta, 0, sizeof(*ena_meta)); + ena_tx_ctx->meta_valid = 1; } else { ena_tx_ctx->meta_valid = 0; } @@ -3004,7 +3010,7 @@ static netdev_tx_t ena_start_xmit(struct sk_buff *skb, struct net_device *dev) ena_tx_ctx.header_len = header_len; /* set flags and meta data */ - ena_tx_csum(&ena_tx_ctx, skb); + ena_tx_csum(&ena_tx_ctx, skb, tx_ring->disable_meta_caching); rc = ena_xmit_common(dev, tx_ring, @@ -3118,7 +3124,9 @@ static void ena_config_host_info(struct ena_com_dev *ena_dev, struct pci_dev *pd host_info->driver_supported_features = ENA_ADMIN_HOST_INFO_RX_OFFSET_MASK | - ENA_ADMIN_HOST_INFO_INTERRUPT_MODERATION_MASK; + ENA_ADMIN_HOST_INFO_INTERRUPT_MODERATION_MASK | + ENA_ADMIN_HOST_INFO_RX_BUF_MIRRORING_MASK | + ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_MASK; rc = ena_com_set_host_attributes(ena_dev); if (rc) { @@ -3271,10 +3279,71 @@ static int ena_device_validate_params(struct ena_adapter *adapter, return 0; } +static void set_default_llq_configurations(struct ena_llq_configurations *llq_config) +{ + llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER; + llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY; + llq_config->llq_num_decs_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2; + llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_128B; + llq_config->llq_ring_entry_size_value = 128; +} + +static int ena_set_queues_placement_policy(struct pci_dev *pdev, + struct ena_com_dev *ena_dev, + struct ena_admin_feature_llq_desc *llq, + struct ena_llq_configurations *llq_default_configurations) +{ + int rc; + u32 llq_feature_mask; + + llq_feature_mask = 1 << ENA_ADMIN_LLQ; + if (!(ena_dev->supported_features & llq_feature_mask)) { + dev_err(&pdev->dev, + "LLQ is not supported Fallback to host mode policy.\n"); + ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; + return 0; + } + + rc = ena_com_config_dev_mode(ena_dev, llq, llq_default_configurations); + if (unlikely(rc)) { + dev_err(&pdev->dev, + "Failed to configure the device mode. Fallback to host mode policy.\n"); + ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; + } + + return 0; +} + +static int ena_map_llq_mem_bar(struct pci_dev *pdev, struct ena_com_dev *ena_dev, + int bars) +{ + bool has_mem_bar = !!(bars & BIT(ENA_MEM_BAR)); + + if (!has_mem_bar) { + if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { + dev_err(&pdev->dev, + "ENA device does not expose LLQ bar. Fallback to host mode policy.\n"); + ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; + } + + return 0; + } + + ena_dev->mem_bar = devm_ioremap_wc(&pdev->dev, + pci_resource_start(pdev, ENA_MEM_BAR), + pci_resource_len(pdev, ENA_MEM_BAR)); + + if (!ena_dev->mem_bar) + return -EFAULT; + + return 0; +} + static int ena_device_init(struct ena_com_dev *ena_dev, struct pci_dev *pdev, struct ena_com_dev_get_features_ctx *get_feat_ctx, bool *wd_state) { + struct ena_llq_configurations llq_config; struct device *dev = &pdev->dev; bool readless_supported; u32 aenq_groups; @@ -3365,6 +3434,15 @@ static int ena_device_init(struct ena_com_dev *ena_dev, struct pci_dev *pdev, *wd_state = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE)); + set_default_llq_configurations(&llq_config); + + rc = ena_set_queues_placement_policy(pdev, ena_dev, &get_feat_ctx->llq, + &llq_config); + if (rc) { + dev_err(&pdev->dev, "ena device init failed\n"); + goto err_admin_init; + } + return 0; err_admin_init: @@ -3871,54 +3949,6 @@ static u32 ena_calc_max_io_queue_num(struct pci_dev *pdev, return max_num_io_queues; } -static int ena_set_queues_placement_policy(struct pci_dev *pdev, - struct ena_com_dev *ena_dev, - struct ena_admin_feature_llq_desc *llq, - struct ena_llq_configurations *llq_default_configurations) -{ - bool has_mem_bar; - int rc; - u32 llq_feature_mask; - - llq_feature_mask = 1 << ENA_ADMIN_LLQ; - if (!(ena_dev->supported_features & llq_feature_mask)) { - dev_err(&pdev->dev, - "LLQ is not supported Fallback to host mode policy.\n"); - ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; - return 0; - } - - has_mem_bar = pci_select_bars(pdev, IORESOURCE_MEM) & BIT(ENA_MEM_BAR); - - rc = ena_com_config_dev_mode(ena_dev, llq, llq_default_configurations); - if (unlikely(rc)) { - dev_err(&pdev->dev, - "Failed to configure the device mode. Fallback to host mode policy.\n"); - ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; - return 0; - } - - /* Nothing to config, exit */ - if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) - return 0; - - if (!has_mem_bar) { - dev_err(&pdev->dev, - "ENA device does not expose LLQ bar. Fallback to host mode policy.\n"); - ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; - return 0; - } - - ena_dev->mem_bar = devm_ioremap_wc(&pdev->dev, - pci_resource_start(pdev, ENA_MEM_BAR), - pci_resource_len(pdev, ENA_MEM_BAR)); - - if (!ena_dev->mem_bar) - return -EFAULT; - - return 0; -} - static void ena_set_dev_offloads(struct ena_com_dev_get_features_ctx *feat, struct net_device *netdev) { @@ -4034,14 +4064,6 @@ static void ena_release_bars(struct ena_com_dev *ena_dev, struct pci_dev *pdev) pci_release_selected_regions(pdev, release_bars); } -static void set_default_llq_configurations(struct ena_llq_configurations *llq_config) -{ - llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER; - llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_128B; - llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY; - llq_config->llq_num_decs_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2; - llq_config->llq_ring_entry_size_value = 128; -} static int ena_calc_io_queue_size(struct ena_calc_queue_size_ctx *ctx) { @@ -4123,7 +4145,6 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { struct ena_calc_queue_size_ctx calc_queue_ctx = { 0 }; struct ena_com_dev_get_features_ctx get_feat_ctx; - struct ena_llq_configurations llq_config; struct ena_com_dev *ena_dev = NULL; struct ena_adapter *adapter; struct net_device *netdev; @@ -4178,13 +4199,10 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto err_free_region; } - set_default_llq_configurations(&llq_config); - - rc = ena_set_queues_placement_policy(pdev, ena_dev, &get_feat_ctx.llq, - &llq_config); + rc = ena_map_llq_mem_bar(pdev, ena_dev, bars); if (rc) { - dev_err(&pdev->dev, "ena device init failed\n"); - goto err_device_destroy; + dev_err(&pdev->dev, "ena llq bar mapping failed\n"); + goto err_free_ena_dev; } calc_queue_ctx.ena_dev = ena_dev; @@ -4241,6 +4259,11 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adapter->xdp_num_queues = 0; adapter->rx_copybreak = ENA_DEFAULT_RX_COPYBREAK; + if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) + adapter->disable_meta_caching = + !!(get_feat_ctx.llq.accel_mode.u.get.supported_flags & + BIT(ENA_ADMIN_DISABLE_META_CACHING)); + adapter->wd_state = wd_state; snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d", adapters_found); @@ -4420,13 +4443,12 @@ static void ena_shutdown(struct pci_dev *pdev) __ena_shutoff(pdev, true); } -#ifdef CONFIG_PM /* ena_suspend - PM suspend callback - * @pdev: PCI device information struct - * @state:power state + * @dev_d: Device information struct */ -static int ena_suspend(struct pci_dev *pdev, pm_message_t state) +static int __maybe_unused ena_suspend(struct device *dev_d) { + struct pci_dev *pdev = to_pci_dev(dev_d); struct ena_adapter *adapter = pci_get_drvdata(pdev); u64_stats_update_begin(&adapter->syncp); @@ -4445,12 +4467,11 @@ static int ena_suspend(struct pci_dev *pdev, pm_message_t state) } /* ena_resume - PM resume callback - * @pdev: PCI device information struct - * + * @dev_d: Device information struct */ -static int ena_resume(struct pci_dev *pdev) +static int __maybe_unused ena_resume(struct device *dev_d) { - struct ena_adapter *adapter = pci_get_drvdata(pdev); + struct ena_adapter *adapter = dev_get_drvdata(dev_d); int rc; u64_stats_update_begin(&adapter->syncp); @@ -4462,7 +4483,8 @@ static int ena_resume(struct pci_dev *pdev) rtnl_unlock(); return rc; } -#endif + +static SIMPLE_DEV_PM_OPS(ena_pm_ops, ena_suspend, ena_resume); static struct pci_driver ena_pci_driver = { .name = DRV_MODULE_NAME, @@ -4470,10 +4492,7 @@ static struct pci_driver ena_pci_driver = { .probe = ena_probe, .remove = ena_remove, .shutdown = ena_shutdown, -#ifdef CONFIG_PM - .suspend = ena_suspend, - .resume = ena_resume, -#endif + .driver.pm = &ena_pm_ops, .sriov_configure = pci_sriov_configure_simple, }; |