From dcb95f06eab84bb3283273a63af8f930bf27c9d7 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 23 Nov 2021 18:18:32 +0100 Subject: e1000: switch to napi_consume_skb() In order to take the best from per-cpu NAPI skbuff_head caches and CPU cycles, let's switch from dev_kfree_skb_any(), which passes skb back to the mm layer, to napi_consume_skb(), which feeds those caches on non-zero budget instead (falls back to the former on 0). Do the replacement in e1000_unmap_and_free_tx_resource(). There are 4 call sites of this function throughout the driver: * e1000_clean_tx_ring(). Slowpath, process context, cleans the whole Tx ring on ifdown. Use budget of 0 here; * e1000_tx_map(). Hotpath, net Tx softirq, unmaps the buffers in case of error. Use 0 as well; * e1000_clean_tx_irq(). Hotpath, NAPI Tx completion polling cycle. As the driver doesn't count completed Tx entries towards the NAPI budget, just use the poll budget of 64 to utilize caches. Apart from being a preparation for switching to napi_build_skb(), this is useful on its own as well, as napi_consume_skb() flushes skb caches by batches of 32 instead of one-at-a-time. Signed-off-by: Alexander Lobakin Reviewed-by: Michal Swiatkowski Tested-by: Tony Brelinski Signed-off-by: Tony Nguyen --- drivers/net/ethernet/intel/e1000/e1000_main.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/net/ethernet/intel/e1000/e1000_main.c b/drivers/net/ethernet/intel/e1000/e1000_main.c index 669060a2e6aa..975a145d48ef 100644 --- a/drivers/net/ethernet/intel/e1000/e1000_main.c +++ b/drivers/net/ethernet/intel/e1000/e1000_main.c @@ -1953,7 +1953,8 @@ void e1000_free_all_tx_resources(struct e1000_adapter *adapter) static void e1000_unmap_and_free_tx_resource(struct e1000_adapter *adapter, - struct e1000_tx_buffer *buffer_info) + struct e1000_tx_buffer *buffer_info, + int budget) { if (buffer_info->dma) { if (buffer_info->mapped_as_page) @@ -1966,7 +1967,7 @@ e1000_unmap_and_free_tx_resource(struct e1000_adapter *adapter, buffer_info->dma = 0; } if (buffer_info->skb) { - dev_kfree_skb_any(buffer_info->skb); + napi_consume_skb(buffer_info->skb, budget); buffer_info->skb = NULL; } buffer_info->time_stamp = 0; @@ -1990,7 +1991,7 @@ static void e1000_clean_tx_ring(struct e1000_adapter *adapter, for (i = 0; i < tx_ring->count; i++) { buffer_info = &tx_ring->buffer_info[i]; - e1000_unmap_and_free_tx_resource(adapter, buffer_info); + e1000_unmap_and_free_tx_resource(adapter, buffer_info, 0); } netdev_reset_queue(adapter->netdev); @@ -2958,7 +2959,7 @@ dma_error: i += tx_ring->count; i--; buffer_info = &tx_ring->buffer_info[i]; - e1000_unmap_and_free_tx_resource(adapter, buffer_info); + e1000_unmap_and_free_tx_resource(adapter, buffer_info, 0); } return 0; @@ -3856,7 +3857,8 @@ static bool e1000_clean_tx_irq(struct e1000_adapter *adapter, } } - e1000_unmap_and_free_tx_resource(adapter, buffer_info); + e1000_unmap_and_free_tx_resource(adapter, buffer_info, + 64); tx_desc->upper.data = 0; if (unlikely(++i == tx_ring->count)) -- cgit v1.2.3 From 89a354c03b2d097f9d580cc48249ca396d027e68 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 23 Nov 2021 18:18:33 +0100 Subject: e1000: switch to napi_build_skb() napi_build_skb() reuses per-cpu NAPI skbuff_head cache in order to save some cycles on freeing/allocating skbuff_heads on every new Rx or completed Tx element. e1000 driver runs Tx completion polling cycle right before the Rx one. Now that e1000 uses napi_consume_skb() to put skbuff_heads of completed entries into the cache, it will never empty and always warm at that moment. Switch to the napi_build_skb() to relax mm pressure on heavy Rx and increase throughput. Signed-off-by: Alexander Lobakin Reviewed-by: Michal Swiatkowski Tested-by: Tony Brelinski Signed-off-by: Tony Nguyen --- drivers/net/ethernet/intel/e1000/e1000_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/intel/e1000/e1000_main.c b/drivers/net/ethernet/intel/e1000/e1000_main.c index 975a145d48ef..3f5feb55cfba 100644 --- a/drivers/net/ethernet/intel/e1000/e1000_main.c +++ b/drivers/net/ethernet/intel/e1000/e1000_main.c @@ -4384,7 +4384,7 @@ static bool e1000_clean_rx_irq(struct e1000_adapter *adapter, if (!skb) { unsigned int frag_len = e1000_frag_len(adapter); - skb = build_skb(data - E1000_HEADROOM, frag_len); + skb = napi_build_skb(data - E1000_HEADROOM, frag_len); if (!skb) { adapter->alloc_rx_buff_failed++; break; -- cgit v1.2.3 From 6e19cf7d3815c009bfd213832a6feac4bafc37ae Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 23 Nov 2021 18:18:34 +0100 Subject: i40e: switch to napi_build_skb() napi_build_skb() reuses per-cpu NAPI skbuff_head cache in order to save some cycles on freeing/allocating skbuff_heads on every new Rx or completed Tx. i40e driver runs Tx completion polling cycle right before the Rx one and uses napi_consume_skb() to feed the cache with skbuff_heads of completed entries, so it's never empty and always warm at that moment. Switch to the napi_build_skb() to relax mm pressure on heavy Rx. Signed-off-by: Alexander Lobakin Reviewed-by: Michal Swiatkowski Tested-by: Gurucharan G Signed-off-by: Tony Nguyen --- drivers/net/ethernet/intel/i40e/i40e_txrx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c index 10a83e5385c7..9e3991caa5c9 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c @@ -2204,7 +2204,7 @@ static struct sk_buff *i40e_build_skb(struct i40e_ring *rx_ring, net_prefetch(xdp->data_meta); /* build an skb around the page buffer */ - skb = build_skb(xdp->data_hard_start, truesize); + skb = napi_build_skb(xdp->data_hard_start, truesize); if (unlikely(!skb)) return NULL; -- cgit v1.2.3 From ef687d61e0e999575c8d781e1edd04b4a388440f Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 23 Nov 2021 18:18:35 +0100 Subject: iavf: switch to napi_build_skb() napi_build_skb() reuses per-cpu NAPI skbuff_head cache in order to save some cycles on freeing/allocating skbuff_heads on every new Rx or completed Tx. iavf driver runs Tx completion polling cycle right before the Rx one and uses napi_consume_skb() to feed the cache with skbuff_heads of completed entries, so it's never empty and always warm at that moment. Switch to the napi_build_skb() to relax mm pressure on heavy Rx. Signed-off-by: Alexander Lobakin Reviewed-by: Michal Swiatkowski Tested-by: Konrad Jankowski Signed-off-by: Tony Nguyen --- drivers/net/ethernet/intel/iavf/iavf_txrx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/intel/iavf/iavf_txrx.c b/drivers/net/ethernet/intel/iavf/iavf_txrx.c index a0b1c18a3273..8cbe7ad1347c 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_txrx.c +++ b/drivers/net/ethernet/intel/iavf/iavf_txrx.c @@ -1366,7 +1366,7 @@ static struct sk_buff *iavf_build_skb(struct iavf_ring *rx_ring, net_prefetch(va); /* build an skb around the page buffer */ - skb = build_skb(va - IAVF_SKB_PAD, truesize); + skb = napi_build_skb(va - IAVF_SKB_PAD, truesize); if (unlikely(!skb)) return NULL; -- cgit v1.2.3 From 5ce666315848a4cb44485d8cd2c5887998da6606 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 23 Nov 2021 18:18:36 +0100 Subject: ice: switch to napi_build_skb() napi_build_skb() reuses per-cpu NAPI skbuff_head cache in order to save some cycles on freeing/allocating skbuff_heads on every new Rx or completed Tx. ice driver runs Tx completion polling cycle right before the Rx one and uses napi_consume_skb() to feed the cache with skbuff_heads of completed entries, so it's never empty and always warm at that moment. Switch to the napi_build_skb() to relax mm pressure on heavy Rx. Signed-off-by: Alexander Lobakin Reviewed-by: Michal Swiatkowski Tested-by: Gurucharan G Signed-off-by: Tony Nguyen --- drivers/net/ethernet/intel/ice/ice_txrx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.c b/drivers/net/ethernet/intel/ice/ice_txrx.c index 47057010b172..486fc5509d13 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.c +++ b/drivers/net/ethernet/intel/ice/ice_txrx.c @@ -948,7 +948,7 @@ ice_build_skb(struct ice_rx_ring *rx_ring, struct ice_rx_buf *rx_buf, */ net_prefetch(xdp->data_meta); /* build an skb around the page buffer */ - skb = build_skb(xdp->data_hard_start, truesize); + skb = napi_build_skb(xdp->data_hard_start, truesize); if (unlikely(!skb)) return NULL; -- cgit v1.2.3 From fa441f0fa8bc4d8231734aa8512d55dfb74fb598 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 23 Nov 2021 18:18:37 +0100 Subject: igb: switch to napi_build_skb() napi_build_skb() reuses per-cpu NAPI skbuff_head cache in order to save some cycles on freeing/allocating skbuff_heads on every new Rx or completed Tx. igb driver runs Tx completion polling cycle right before the Rx one and uses napi_consume_skb() to feed the cache with skbuff_heads of completed entries, so it's never empty and always warm at that moment. Switch to the napi_build_skb() to relax mm pressure on heavy Rx. Signed-off-by: Alexander Lobakin Reviewed-by: Michal Swiatkowski Tested-by: Gurucharan G Signed-off-by: Tony Nguyen --- drivers/net/ethernet/intel/igb/igb_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c index ffe457e395a1..47ece02fce99 100644 --- a/drivers/net/ethernet/intel/igb/igb_main.c +++ b/drivers/net/ethernet/intel/igb/igb_main.c @@ -8367,7 +8367,7 @@ static struct sk_buff *igb_build_skb(struct igb_ring *rx_ring, net_prefetch(xdp->data_meta); /* build an skb around the page buffer */ - skb = build_skb(xdp->data_hard_start, truesize); + skb = napi_build_skb(xdp->data_hard_start, truesize); if (unlikely(!skb)) return NULL; -- cgit v1.2.3 From 4dd330a7e8944ee4e3409d4c99c03f247def9c3b Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 23 Nov 2021 18:18:38 +0100 Subject: igc: switch to napi_build_skb() napi_build_skb() reuses per-cpu NAPI skbuff_head cache in order to save some cycles on freeing/allocating skbuff_heads on every new Rx or completed Tx. igc driver runs Tx completion polling cycle right before the Rx one and uses napi_consume_skb() to feed the cache with skbuff_heads of completed entries, so it's never empty and always warm at that moment. Switch to the napi_build_skb() to relax mm pressure on heavy Rx. Signed-off-by: Alexander Lobakin Reviewed-by: Michal Swiatkowski Tested-by: Nechama Kraus Signed-off-by: Tony Nguyen --- drivers/net/ethernet/intel/igc/igc_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 142c57b7a451..8af267512afa 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -1729,7 +1729,7 @@ static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring, net_prefetch(xdp->data_meta); /* build an skb around the page buffer */ - skb = build_skb(xdp->data_hard_start, truesize); + skb = napi_build_skb(xdp->data_hard_start, truesize); if (unlikely(!skb)) return NULL; -- cgit v1.2.3 From a39363367a377ceda6b58ec15edf07fae1d61b1b Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 23 Nov 2021 18:18:39 +0100 Subject: ixgbe: switch to napi_build_skb() napi_build_skb() reuses per-cpu NAPI skbuff_head cache in order to save some cycles on freeing/allocating skbuff_heads on every new Rx or completed Tx. ixgbe driver runs Tx completion polling cycle right before the Rx one and uses napi_consume_skb() to feed the cache with skbuff_heads of completed entries, so it's never empty and always warm at that moment. Switch to the napi_build_skb() to relax mm pressure on heavy Rx. Signed-off-by: Alexander Lobakin Reviewed-by: Michal Swiatkowski Tested-by: Gurucharan G Signed-off-by: Tony Nguyen --- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 45e2ec4d264d..15095ac868d5 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -2170,7 +2170,7 @@ static struct sk_buff *ixgbe_build_skb(struct ixgbe_ring *rx_ring, net_prefetch(xdp->data_meta); /* build an skb to around the page buffer */ - skb = build_skb(xdp->data_hard_start, truesize); + skb = napi_build_skb(xdp->data_hard_start, truesize); if (unlikely(!skb)) return NULL; -- cgit v1.2.3 From c15500198916cb6f553b43f63070990b8d079b49 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 23 Nov 2021 18:18:40 +0100 Subject: ixgbevf: switch to napi_build_skb() napi_build_skb() reuses per-cpu NAPI skbuff_head cache in order to save some cycles on freeing/allocating skbuff_heads on every new Rx or completed Tx. ixgbevf driver runs Tx completion polling cycle right before the Rx one and uses napi_consume_skb() to feed the cache with skbuff_heads of completed entries, so it's never empty and always warm at that moment. Switch to the napi_build_skb() to relax mm pressure on heavy Rx. Signed-off-by: Alexander Lobakin Reviewed-by: Michal Swiatkowski Tested-by: Konrad Jankowski Signed-off-by: Tony Nguyen --- drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index b1dfbaff8b31..ea73fb3026bc 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -944,7 +944,7 @@ static struct sk_buff *ixgbevf_build_skb(struct ixgbevf_ring *rx_ring, net_prefetch(xdp->data_meta); /* build an skb around the page buffer */ - skb = build_skb(xdp->data_hard_start, truesize); + skb = napi_build_skb(xdp->data_hard_start, truesize); if (unlikely(!skb)) return NULL; -- cgit v1.2.3