diff options
Diffstat (limited to 'drivers/net/ethernet/amazon/ena')
| -rw-r--r-- | drivers/net/ethernet/amazon/ena/ena_admin_defs.h | 8 | ||||
| -rw-r--r-- | drivers/net/ethernet/amazon/ena/ena_com.c | 175 | ||||
| -rw-r--r-- | drivers/net/ethernet/amazon/ena/ena_com.h | 151 | ||||
| -rw-r--r-- | drivers/net/ethernet/amazon/ena/ena_ethtool.c | 89 | ||||
| -rw-r--r-- | drivers/net/ethernet/amazon/ena/ena_netdev.c | 88 | ||||
| -rw-r--r-- | drivers/net/ethernet/amazon/ena/ena_netdev.h | 8 |
6 files changed, 130 insertions, 389 deletions
diff --git a/drivers/net/ethernet/amazon/ena/ena_admin_defs.h b/drivers/net/ethernet/amazon/ena/ena_admin_defs.h index d19f2ecf8e84..8baf847e8622 100644 --- a/drivers/net/ethernet/amazon/ena/ena_admin_defs.h +++ b/drivers/net/ethernet/amazon/ena/ena_admin_defs.h @@ -808,6 +808,12 @@ struct ena_admin_host_info { u16 num_cpus; u16 reserved; + + /* 1 :0 : reserved + * 2 : interrupt_moderation + * 31:3 : reserved + */ + u32 driver_supported_features; }; struct ena_admin_rss_ind_table_entry { @@ -1110,6 +1116,8 @@ struct ena_admin_ena_mmio_req_read_less_resp { #define ENA_ADMIN_HOST_INFO_DEVICE_MASK GENMASK(7, 3) #define ENA_ADMIN_HOST_INFO_BUS_SHIFT 8 #define ENA_ADMIN_HOST_INFO_BUS_MASK GENMASK(15, 8) +#define ENA_ADMIN_HOST_INFO_INTERRUPT_MODERATION_SHIFT 2 +#define ENA_ADMIN_HOST_INFO_INTERRUPT_MODERATION_MASK BIT(2) /* aenq_common_desc */ #define ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK BIT(0) diff --git a/drivers/net/ethernet/amazon/ena/ena_com.c b/drivers/net/ethernet/amazon/ena/ena_com.c index 911a2e7a375a..ea62604fdf8c 100644 --- a/drivers/net/ethernet/amazon/ena/ena_com.c +++ b/drivers/net/ethernet/amazon/ena/ena_com.c @@ -1278,40 +1278,33 @@ static int ena_com_ind_tbl_convert_from_device(struct ena_com_dev *ena_dev) return 0; } -static int ena_com_init_interrupt_moderation_table(struct ena_com_dev *ena_dev) -{ - size_t size; - - size = sizeof(struct ena_intr_moder_entry) * ENA_INTR_MAX_NUM_OF_LEVELS; - - ena_dev->intr_moder_tbl = - devm_kzalloc(ena_dev->dmadev, size, GFP_KERNEL); - if (!ena_dev->intr_moder_tbl) - return -ENOMEM; - - ena_com_config_default_interrupt_moderation_table(ena_dev); - - return 0; -} - static void ena_com_update_intr_delay_resolution(struct ena_com_dev *ena_dev, u16 intr_delay_resolution) { - struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl; - unsigned int i; + /* Initial value of intr_delay_resolution might be 0 */ + u16 prev_intr_delay_resolution = + ena_dev->intr_delay_resolution ? + ena_dev->intr_delay_resolution : + ENA_DEFAULT_INTR_DELAY_RESOLUTION; if (!intr_delay_resolution) { pr_err("Illegal intr_delay_resolution provided. Going to use default 1 usec resolution\n"); - intr_delay_resolution = 1; + intr_delay_resolution = ENA_DEFAULT_INTR_DELAY_RESOLUTION; } - ena_dev->intr_delay_resolution = intr_delay_resolution; /* update Rx */ - for (i = 0; i < ENA_INTR_MAX_NUM_OF_LEVELS; i++) - intr_moder_tbl[i].intr_moder_interval /= intr_delay_resolution; + ena_dev->intr_moder_rx_interval = + ena_dev->intr_moder_rx_interval * + prev_intr_delay_resolution / + intr_delay_resolution; /* update Tx */ - ena_dev->intr_moder_tx_interval /= intr_delay_resolution; + ena_dev->intr_moder_tx_interval = + ena_dev->intr_moder_tx_interval * + prev_intr_delay_resolution / + intr_delay_resolution; + + ena_dev->intr_delay_resolution = intr_delay_resolution; } /*****************************************************************************/ @@ -2776,42 +2769,34 @@ bool ena_com_interrupt_moderation_supported(struct ena_com_dev *ena_dev) ENA_ADMIN_INTERRUPT_MODERATION); } -int ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev, - u32 tx_coalesce_usecs) +static int ena_com_update_nonadaptive_moderation_interval(u32 coalesce_usecs, + u32 intr_delay_resolution, + u32 *intr_moder_interval) { - if (!ena_dev->intr_delay_resolution) { + if (!intr_delay_resolution) { pr_err("Illegal interrupt delay granularity value\n"); return -EFAULT; } - ena_dev->intr_moder_tx_interval = tx_coalesce_usecs / - ena_dev->intr_delay_resolution; + *intr_moder_interval = coalesce_usecs / intr_delay_resolution; return 0; } -int ena_com_update_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev, - u32 rx_coalesce_usecs) +int ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev, + u32 tx_coalesce_usecs) { - if (!ena_dev->intr_delay_resolution) { - pr_err("Illegal interrupt delay granularity value\n"); - return -EFAULT; - } - - /* We use LOWEST entry of moderation table for storing - * nonadaptive interrupt coalescing values - */ - ena_dev->intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval = - rx_coalesce_usecs / ena_dev->intr_delay_resolution; - - return 0; + return ena_com_update_nonadaptive_moderation_interval(tx_coalesce_usecs, + ena_dev->intr_delay_resolution, + &ena_dev->intr_moder_tx_interval); } -void ena_com_destroy_interrupt_moderation(struct ena_com_dev *ena_dev) +int ena_com_update_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev, + u32 rx_coalesce_usecs) { - if (ena_dev->intr_moder_tbl) - devm_kfree(ena_dev->dmadev, ena_dev->intr_moder_tbl); - ena_dev->intr_moder_tbl = NULL; + return ena_com_update_nonadaptive_moderation_interval(rx_coalesce_usecs, + ena_dev->intr_delay_resolution, + &ena_dev->intr_moder_rx_interval); } int ena_com_init_interrupt_moderation(struct ena_com_dev *ena_dev) @@ -2838,66 +2823,14 @@ int ena_com_init_interrupt_moderation(struct ena_com_dev *ena_dev) return rc; } - rc = ena_com_init_interrupt_moderation_table(ena_dev); - if (rc) - goto err; - /* if moderation is supported by device we set adaptive moderation */ delay_resolution = get_resp.u.intr_moderation.intr_delay_resolution; ena_com_update_intr_delay_resolution(ena_dev, delay_resolution); - /* Disable adaptive moderation by default - can be enabled from - * ethtool - */ + /* Disable adaptive moderation by default - can be enabled later */ ena_com_disable_adaptive_moderation(ena_dev); return 0; -err: - ena_com_destroy_interrupt_moderation(ena_dev); - return rc; -} - -void ena_com_config_default_interrupt_moderation_table(struct ena_com_dev *ena_dev) -{ - struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl; - - if (!intr_moder_tbl) - return; - - intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval = - ENA_INTR_LOWEST_USECS; - intr_moder_tbl[ENA_INTR_MODER_LOWEST].pkts_per_interval = - ENA_INTR_LOWEST_PKTS; - intr_moder_tbl[ENA_INTR_MODER_LOWEST].bytes_per_interval = - ENA_INTR_LOWEST_BYTES; - - intr_moder_tbl[ENA_INTR_MODER_LOW].intr_moder_interval = - ENA_INTR_LOW_USECS; - intr_moder_tbl[ENA_INTR_MODER_LOW].pkts_per_interval = - ENA_INTR_LOW_PKTS; - intr_moder_tbl[ENA_INTR_MODER_LOW].bytes_per_interval = - ENA_INTR_LOW_BYTES; - - intr_moder_tbl[ENA_INTR_MODER_MID].intr_moder_interval = - ENA_INTR_MID_USECS; - intr_moder_tbl[ENA_INTR_MODER_MID].pkts_per_interval = - ENA_INTR_MID_PKTS; - intr_moder_tbl[ENA_INTR_MODER_MID].bytes_per_interval = - ENA_INTR_MID_BYTES; - - intr_moder_tbl[ENA_INTR_MODER_HIGH].intr_moder_interval = - ENA_INTR_HIGH_USECS; - intr_moder_tbl[ENA_INTR_MODER_HIGH].pkts_per_interval = - ENA_INTR_HIGH_PKTS; - intr_moder_tbl[ENA_INTR_MODER_HIGH].bytes_per_interval = - ENA_INTR_HIGH_BYTES; - - intr_moder_tbl[ENA_INTR_MODER_HIGHEST].intr_moder_interval = - ENA_INTR_HIGHEST_USECS; - intr_moder_tbl[ENA_INTR_MODER_HIGHEST].pkts_per_interval = - ENA_INTR_HIGHEST_PKTS; - intr_moder_tbl[ENA_INTR_MODER_HIGHEST].bytes_per_interval = - ENA_INTR_HIGHEST_BYTES; } unsigned int ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev) @@ -2907,49 +2840,7 @@ unsigned int ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev * unsigned int ena_com_get_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev) { - struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl; - - if (intr_moder_tbl) - return intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval; - - return 0; -} - -void ena_com_init_intr_moderation_entry(struct ena_com_dev *ena_dev, - enum ena_intr_moder_level level, - struct ena_intr_moder_entry *entry) -{ - struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl; - - if (level >= ENA_INTR_MAX_NUM_OF_LEVELS) - return; - - intr_moder_tbl[level].intr_moder_interval = entry->intr_moder_interval; - if (ena_dev->intr_delay_resolution) - intr_moder_tbl[level].intr_moder_interval /= - ena_dev->intr_delay_resolution; - intr_moder_tbl[level].pkts_per_interval = entry->pkts_per_interval; - - /* use hardcoded value until ethtool supports bytecount parameter */ - if (entry->bytes_per_interval != ENA_INTR_BYTE_COUNT_NOT_SUPPORTED) - intr_moder_tbl[level].bytes_per_interval = entry->bytes_per_interval; -} - -void ena_com_get_intr_moderation_entry(struct ena_com_dev *ena_dev, - enum ena_intr_moder_level level, - struct ena_intr_moder_entry *entry) -{ - struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl; - - if (level >= ENA_INTR_MAX_NUM_OF_LEVELS) - return; - - entry->intr_moder_interval = intr_moder_tbl[level].intr_moder_interval; - if (ena_dev->intr_delay_resolution) - entry->intr_moder_interval *= ena_dev->intr_delay_resolution; - entry->pkts_per_interval = - intr_moder_tbl[level].pkts_per_interval; - entry->bytes_per_interval = intr_moder_tbl[level].bytes_per_interval; + return ena_dev->intr_moder_rx_interval; } int ena_com_config_dev_mode(struct ena_com_dev *ena_dev, diff --git a/drivers/net/ethernet/amazon/ena/ena_com.h b/drivers/net/ethernet/amazon/ena/ena_com.h index 0d3664fe260d..7c941eba0bc9 100644 --- a/drivers/net/ethernet/amazon/ena/ena_com.h +++ b/drivers/net/ethernet/amazon/ena/ena_com.h @@ -72,46 +72,14 @@ /*****************************************************************************/ /* ENA adaptive interrupt moderation settings */ -#define ENA_INTR_LOWEST_USECS (0) -#define ENA_INTR_LOWEST_PKTS (3) -#define ENA_INTR_LOWEST_BYTES (2 * 1524) - -#define ENA_INTR_LOW_USECS (32) -#define ENA_INTR_LOW_PKTS (12) -#define ENA_INTR_LOW_BYTES (16 * 1024) - -#define ENA_INTR_MID_USECS (80) -#define ENA_INTR_MID_PKTS (48) -#define ENA_INTR_MID_BYTES (64 * 1024) - -#define ENA_INTR_HIGH_USECS (128) -#define ENA_INTR_HIGH_PKTS (96) -#define ENA_INTR_HIGH_BYTES (128 * 1024) - -#define ENA_INTR_HIGHEST_USECS (192) -#define ENA_INTR_HIGHEST_PKTS (128) -#define ENA_INTR_HIGHEST_BYTES (192 * 1024) - #define ENA_INTR_INITIAL_TX_INTERVAL_USECS 196 -#define ENA_INTR_INITIAL_RX_INTERVAL_USECS 4 -#define ENA_INTR_DELAY_OLD_VALUE_WEIGHT 6 -#define ENA_INTR_DELAY_NEW_VALUE_WEIGHT 4 -#define ENA_INTR_MODER_LEVEL_STRIDE 2 -#define ENA_INTR_BYTE_COUNT_NOT_SUPPORTED 0xFFFFFF +#define ENA_INTR_INITIAL_RX_INTERVAL_USECS 0 +#define ENA_DEFAULT_INTR_DELAY_RESOLUTION 1 #define ENA_HW_HINTS_NO_TIMEOUT 0xFFFF #define ENA_FEATURE_MAX_QUEUE_EXT_VER 1 -enum ena_intr_moder_level { - ENA_INTR_MODER_LOWEST = 0, - ENA_INTR_MODER_LOW, - ENA_INTR_MODER_MID, - ENA_INTR_MODER_HIGH, - ENA_INTR_MODER_HIGHEST, - ENA_INTR_MAX_NUM_OF_LEVELS, -}; - struct ena_llq_configurations { enum ena_admin_llq_header_location llq_header_location; enum ena_admin_llq_ring_entry_size llq_ring_entry_size; @@ -120,12 +88,6 @@ struct ena_llq_configurations { u16 llq_ring_entry_size_value; }; -struct ena_intr_moder_entry { - unsigned int intr_moder_interval; - unsigned int pkts_per_interval; - unsigned int bytes_per_interval; -}; - enum queue_direction { ENA_COM_IO_QUEUE_DIRECTION_TX, ENA_COM_IO_QUEUE_DIRECTION_RX @@ -376,7 +338,13 @@ struct ena_com_dev { struct ena_host_attribute host_attr; bool adaptive_coalescing; u16 intr_delay_resolution; + + /* interrupt moderation intervals are in usec divided by + * intr_delay_resolution, which is supplied by the device. + */ u32 intr_moder_tx_interval; + u32 intr_moder_rx_interval; + struct ena_intr_moder_entry *intr_moder_tbl; struct ena_com_llq_info llq_info; @@ -914,11 +882,6 @@ int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue, */ int ena_com_init_interrupt_moderation(struct ena_com_dev *ena_dev); -/* ena_com_destroy_interrupt_moderation - Destroy interrupt moderation resources - * @ena_dev: ENA communication layer struct - */ -void ena_com_destroy_interrupt_moderation(struct ena_com_dev *ena_dev); - /* ena_com_interrupt_moderation_supported - Return if interrupt moderation * capability is supported by the device. * @@ -926,12 +889,6 @@ void ena_com_destroy_interrupt_moderation(struct ena_com_dev *ena_dev); */ bool ena_com_interrupt_moderation_supported(struct ena_com_dev *ena_dev); -/* ena_com_config_default_interrupt_moderation_table - Restore the interrupt - * moderation table back to the default parameters. - * @ena_dev: ENA communication layer struct - */ -void ena_com_config_default_interrupt_moderation_table(struct ena_com_dev *ena_dev); - /* ena_com_update_nonadaptive_moderation_interval_tx - Update the * non-adaptive interval in Tx direction. * @ena_dev: ENA communication layer struct @@ -968,29 +925,6 @@ unsigned int ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev * */ unsigned int ena_com_get_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev); -/* ena_com_init_intr_moderation_entry - Update a single entry in the interrupt - * moderation table. - * @ena_dev: ENA communication layer struct - * @level: Interrupt moderation table level - * @entry: Entry value - * - * Update a single entry in the interrupt moderation table. - */ -void ena_com_init_intr_moderation_entry(struct ena_com_dev *ena_dev, - enum ena_intr_moder_level level, - struct ena_intr_moder_entry *entry); - -/* ena_com_get_intr_moderation_entry - Init ena_intr_moder_entry. - * @ena_dev: ENA communication layer struct - * @level: Interrupt moderation table level - * @entry: Entry to fill. - * - * Initialize the entry according to the adaptive interrupt moderation table. - */ -void ena_com_get_intr_moderation_entry(struct ena_com_dev *ena_dev, - enum ena_intr_moder_level level, - struct ena_intr_moder_entry *entry); - /* ena_com_config_dev_mode - Configure the placement policy of the device. * @ena_dev: ENA communication layer struct * @llq_features: LLQ feature descriptor, retrieve via @@ -1016,75 +950,6 @@ static inline void ena_com_disable_adaptive_moderation(struct ena_com_dev *ena_d ena_dev->adaptive_coalescing = false; } -/* ena_com_calculate_interrupt_delay - Calculate new interrupt delay - * @ena_dev: ENA communication layer struct - * @pkts: Number of packets since the last update - * @bytes: Number of bytes received since the last update. - * @smoothed_interval: Returned interval - * @moder_tbl_idx: Current table level as input update new level as return - * value. - */ -static inline void ena_com_calculate_interrupt_delay(struct ena_com_dev *ena_dev, - unsigned int pkts, - unsigned int bytes, - unsigned int *smoothed_interval, - unsigned int *moder_tbl_idx) -{ - enum ena_intr_moder_level curr_moder_idx, new_moder_idx; - struct ena_intr_moder_entry *curr_moder_entry; - struct ena_intr_moder_entry *pred_moder_entry; - struct ena_intr_moder_entry *new_moder_entry; - struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl; - unsigned int interval; - - /* We apply adaptive moderation on Rx path only. - * Tx uses static interrupt moderation. - */ - if (!pkts || !bytes) - /* Tx interrupt, or spurious interrupt, - * in both cases we just use same delay values - */ - return; - - curr_moder_idx = (enum ena_intr_moder_level)(*moder_tbl_idx); - if (unlikely(curr_moder_idx >= ENA_INTR_MAX_NUM_OF_LEVELS)) { - pr_err("Wrong moderation index %u\n", curr_moder_idx); - return; - } - - curr_moder_entry = &intr_moder_tbl[curr_moder_idx]; - new_moder_idx = curr_moder_idx; - - if (curr_moder_idx == ENA_INTR_MODER_LOWEST) { - if ((pkts > curr_moder_entry->pkts_per_interval) || - (bytes > curr_moder_entry->bytes_per_interval)) - new_moder_idx = - (enum ena_intr_moder_level)(curr_moder_idx + ENA_INTR_MODER_LEVEL_STRIDE); - } else { - pred_moder_entry = &intr_moder_tbl[curr_moder_idx - ENA_INTR_MODER_LEVEL_STRIDE]; - - if ((pkts <= pred_moder_entry->pkts_per_interval) || - (bytes <= pred_moder_entry->bytes_per_interval)) - new_moder_idx = - (enum ena_intr_moder_level)(curr_moder_idx - ENA_INTR_MODER_LEVEL_STRIDE); - else if ((pkts > curr_moder_entry->pkts_per_interval) || - (bytes > curr_moder_entry->bytes_per_interval)) { - if (curr_moder_idx != ENA_INTR_MODER_HIGHEST) - new_moder_idx = - (enum ena_intr_moder_level)(curr_moder_idx + ENA_INTR_MODER_LEVEL_STRIDE); - } - } - new_moder_entry = &intr_moder_tbl[new_moder_idx]; - - interval = new_moder_entry->intr_moder_interval; - *smoothed_interval = ( - (interval * ENA_INTR_DELAY_NEW_VALUE_WEIGHT + - ENA_INTR_DELAY_OLD_VALUE_WEIGHT * (*smoothed_interval)) + 5) / - 10; - - *moder_tbl_idx = new_moder_idx; -} - /* ena_com_update_intr_reg - Prepare interrupt register * @intr_reg: interrupt register to update. * @rx_delay_interval: Rx interval in usecs diff --git a/drivers/net/ethernet/amazon/ena/ena_ethtool.c b/drivers/net/ethernet/amazon/ena/ena_ethtool.c index b997c3ce9e2b..16553d92fad2 100644 --- a/drivers/net/ethernet/amazon/ena/ena_ethtool.c +++ b/drivers/net/ethernet/amazon/ena/ena_ethtool.c @@ -305,32 +305,21 @@ static int ena_get_coalesce(struct net_device *net_dev, { struct ena_adapter *adapter = netdev_priv(net_dev); struct ena_com_dev *ena_dev = adapter->ena_dev; - struct ena_intr_moder_entry intr_moder_entry; if (!ena_com_interrupt_moderation_supported(ena_dev)) { /* the devie doesn't support interrupt moderation */ return -EOPNOTSUPP; } + coalesce->tx_coalesce_usecs = - ena_com_get_nonadaptive_moderation_interval_tx(ena_dev) / + ena_com_get_nonadaptive_moderation_interval_tx(ena_dev) * ena_dev->intr_delay_resolution; - if (!ena_com_get_adaptive_moderation_enabled(ena_dev)) { + + if (!ena_com_get_adaptive_moderation_enabled(ena_dev)) coalesce->rx_coalesce_usecs = ena_com_get_nonadaptive_moderation_interval_rx(ena_dev) - / ena_dev->intr_delay_resolution; - } else { - ena_com_get_intr_moderation_entry(adapter->ena_dev, ENA_INTR_MODER_LOWEST, &intr_moder_entry); - coalesce->rx_coalesce_usecs_low = intr_moder_entry.intr_moder_interval; - coalesce->rx_max_coalesced_frames_low = intr_moder_entry.pkts_per_interval; - - ena_com_get_intr_moderation_entry(adapter->ena_dev, ENA_INTR_MODER_MID, &intr_moder_entry); - coalesce->rx_coalesce_usecs = intr_moder_entry.intr_moder_interval; - coalesce->rx_max_coalesced_frames = intr_moder_entry.pkts_per_interval; - - ena_com_get_intr_moderation_entry(adapter->ena_dev, ENA_INTR_MODER_HIGHEST, &intr_moder_entry); - coalesce->rx_coalesce_usecs_high = intr_moder_entry.intr_moder_interval; - coalesce->rx_max_coalesced_frames_high = intr_moder_entry.pkts_per_interval; - } + * ena_dev->intr_delay_resolution; + coalesce->use_adaptive_rx_coalesce = ena_com_get_adaptive_moderation_enabled(ena_dev); @@ -348,12 +337,22 @@ static void ena_update_tx_rings_intr_moderation(struct ena_adapter *adapter) adapter->tx_ring[i].smoothed_interval = val; } +static void ena_update_rx_rings_intr_moderation(struct ena_adapter *adapter) +{ + unsigned int val; + int i; + + val = ena_com_get_nonadaptive_moderation_interval_rx(adapter->ena_dev); + + for (i = 0; i < adapter->num_queues; i++) + adapter->rx_ring[i].smoothed_interval = val; +} + static int ena_set_coalesce(struct net_device *net_dev, struct ethtool_coalesce *coalesce) { struct ena_adapter *adapter = netdev_priv(net_dev); struct ena_com_dev *ena_dev = adapter->ena_dev; - struct ena_intr_moder_entry intr_moder_entry; int rc; if (!ena_com_interrupt_moderation_supported(ena_dev)) { @@ -361,22 +360,6 @@ static int ena_set_coalesce(struct net_device *net_dev, return -EOPNOTSUPP; } - if (coalesce->rx_coalesce_usecs_irq || - coalesce->rx_max_coalesced_frames_irq || - coalesce->tx_coalesce_usecs_irq || - coalesce->tx_max_coalesced_frames || - coalesce->tx_max_coalesced_frames_irq || - coalesce->stats_block_coalesce_usecs || - coalesce->use_adaptive_tx_coalesce || - coalesce->pkt_rate_low || - coalesce->tx_coalesce_usecs_low || - coalesce->tx_max_coalesced_frames_low || - coalesce->pkt_rate_high || - coalesce->tx_coalesce_usecs_high || - coalesce->tx_max_coalesced_frames_high || - coalesce->rate_sample_interval) - return -EINVAL; - rc = ena_com_update_nonadaptive_moderation_interval_tx(ena_dev, coalesce->tx_coalesce_usecs); if (rc) @@ -384,37 +367,23 @@ static int ena_set_coalesce(struct net_device *net_dev, ena_update_tx_rings_intr_moderation(adapter); - if (ena_com_get_adaptive_moderation_enabled(ena_dev)) { - if (!coalesce->use_adaptive_rx_coalesce) { - ena_com_disable_adaptive_moderation(ena_dev); - rc = ena_com_update_nonadaptive_moderation_interval_rx(ena_dev, - coalesce->rx_coalesce_usecs); - return rc; - } - } else { /* was in non-adaptive mode */ - if (coalesce->use_adaptive_rx_coalesce) { + if (coalesce->use_adaptive_rx_coalesce) { + if (!ena_com_get_adaptive_moderation_enabled(ena_dev)) ena_com_enable_adaptive_moderation(ena_dev); - } else { - rc = ena_com_update_nonadaptive_moderation_interval_rx(ena_dev, - coalesce->rx_coalesce_usecs); - return rc; - } + return 0; } - intr_moder_entry.intr_moder_interval = coalesce->rx_coalesce_usecs_low; - intr_moder_entry.pkts_per_interval = coalesce->rx_max_coalesced_frames_low; - intr_moder_entry.bytes_per_interval = ENA_INTR_BYTE_COUNT_NOT_SUPPORTED; - ena_com_init_intr_moderation_entry(adapter->ena_dev, ENA_INTR_MODER_LOWEST, &intr_moder_entry); + rc = ena_com_update_nonadaptive_moderation_interval_rx(ena_dev, + coalesce->rx_coalesce_usecs); + if (rc) + return rc; - intr_moder_entry.intr_moder_interval = coalesce->rx_coalesce_usecs; - intr_moder_entry.pkts_per_interval = coalesce->rx_max_coalesced_frames; - intr_moder_entry.bytes_per_interval = ENA_INTR_BYTE_COUNT_NOT_SUPPORTED; - ena_com_init_intr_moderation_entry(adapter->ena_dev, ENA_INTR_MODER_MID, &intr_moder_entry); + ena_update_rx_rings_intr_moderation(adapter); - intr_moder_entry.intr_moder_interval = coalesce->rx_coalesce_usecs_high; - intr_moder_entry.pkts_per_interval = coalesce->rx_max_coalesced_frames_high; - intr_moder_entry.bytes_per_interval = ENA_INTR_BYTE_COUNT_NOT_SUPPORTED; - ena_com_init_intr_moderation_entry(adapter->ena_dev, ENA_INTR_MODER_HIGHEST, &intr_moder_entry); + if (!coalesce->use_adaptive_rx_coalesce) { + if (ena_com_get_adaptive_moderation_enabled(ena_dev)) + ena_com_disable_adaptive_moderation(ena_dev); + } return 0; } diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c index 664e3ed97ea9..c487d2a7d6dd 100644 --- a/drivers/net/ethernet/amazon/ena/ena_netdev.c +++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c @@ -158,7 +158,6 @@ static void ena_init_io_rings_common(struct ena_adapter *adapter, ring->adapter = adapter; ring->ena_dev = adapter->ena_dev; ring->per_napi_packets = 0; - ring->per_napi_bytes = 0; ring->cpu = 0; ring->first_interrupt = false; ring->no_interrupt_event_cnt = 0; @@ -196,6 +195,7 @@ static void ena_init_io_rings(struct ena_adapter *adapter) rxr->smoothed_interval = ena_com_get_nonadaptive_moderation_interval_rx(ena_dev); rxr->empty_rx_queue = 0; + adapter->ena_napi[i].dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; } } @@ -712,6 +712,7 @@ static void ena_destroy_all_rx_queues(struct ena_adapter *adapter) for (i = 0; i < adapter->num_queues; i++) { ena_qid = ENA_IO_RXQ_IDX(i); + cancel_work_sync(&adapter->ena_napi[i].dim.work); ena_com_destroy_io_queue(adapter->ena_dev, ena_qid); } } @@ -823,7 +824,8 @@ static int ena_clean_tx_irq(struct ena_ring *tx_ring, u32 budget) above_thresh = ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq, ENA_TX_WAKEUP_THRESH); - if (netif_tx_queue_stopped(txq) && above_thresh) { + if (netif_tx_queue_stopped(txq) && above_thresh && + test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags)) { netif_tx_wake_queue(txq); u64_stats_update_begin(&tx_ring->syncp); tx_ring->tx_stats.queue_wakeup++; @@ -832,9 +834,6 @@ static int ena_clean_tx_irq(struct ena_ring *tx_ring, u32 budget) __netif_tx_unlock(txq); } - tx_ring->per_napi_bytes += tx_bytes; - tx_ring->per_napi_packets += tx_pkts; - return tx_pkts; } @@ -1118,7 +1117,6 @@ static int ena_clean_rx_irq(struct ena_ring *rx_ring, struct napi_struct *napi, } while (likely(res_budget)); work_done = budget - res_budget; - rx_ring->per_napi_bytes += total_len; rx_ring->per_napi_packets += work_done; u64_stats_update_begin(&rx_ring->syncp); rx_ring->rx_stats.bytes += total_len; @@ -1155,35 +1153,50 @@ error: return 0; } -void ena_adjust_intr_moderation(struct ena_ring *rx_ring, - struct ena_ring *tx_ring) +static void ena_dim_work(struct work_struct *w) { - /* We apply adaptive moderation on Rx path only. - * Tx uses static interrupt moderation. - */ - ena_com_calculate_interrupt_delay(rx_ring->ena_dev, - rx_ring->per_napi_packets, - rx_ring->per_napi_bytes, - &rx_ring->smoothed_interval, - &rx_ring->moder_tbl_idx); - - /* Reset per napi packets/bytes */ - tx_ring->per_napi_packets = 0; - tx_ring->per_napi_bytes = 0; + struct dim *dim = container_of(w, struct dim, work); + struct dim_cq_moder cur_moder = + net_dim_get_rx_moderation(dim->mode, dim->profile_ix); + struct ena_napi *ena_napi = container_of(dim, struct ena_napi, dim); + + ena_napi->rx_ring->smoothed_interval = cur_moder.usec; + dim->state = DIM_START_MEASURE; +} + +static void ena_adjust_adaptive_rx_intr_moderation(struct ena_napi *ena_napi) +{ + struct dim_sample dim_sample; + struct ena_ring *rx_ring = ena_napi->rx_ring; + + if (!rx_ring->per_napi_packets) + return; + + rx_ring->non_empty_napi_events++; + + dim_update_sample(rx_ring->non_empty_napi_events, + rx_ring->rx_stats.cnt, + rx_ring->rx_stats.bytes, + &dim_sample); + + net_dim(&ena_napi->dim, dim_sample); + rx_ring->per_napi_packets = 0; - rx_ring->per_napi_bytes = 0; } static void ena_unmask_interrupt(struct ena_ring *tx_ring, struct ena_ring *rx_ring) { struct ena_eth_io_intr_reg intr_reg; + u32 rx_interval = ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev) ? + rx_ring->smoothed_interval : + ena_com_get_nonadaptive_moderation_interval_rx(rx_ring->ena_dev); /* Update intr register: rx intr delay, * tx intr delay and interrupt unmask */ ena_com_update_intr_reg(&intr_reg, - rx_ring->smoothed_interval, + rx_interval, tx_ring->smoothed_interval, true); @@ -1260,9 +1273,11 @@ static int ena_io_poll(struct napi_struct *napi, int budget) * from the interrupt context (vs from sk_busy_loop) */ if (napi_complete_done(napi, rx_work_done)) { - /* Tx and Rx share the same interrupt vector */ + /* We apply adaptive moderation on Rx path only. + * Tx uses static interrupt moderation. + */ if (ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev)) - ena_adjust_intr_moderation(rx_ring, tx_ring); + ena_adjust_adaptive_rx_intr_moderation(ena_napi); ena_unmask_interrupt(tx_ring, rx_ring); } @@ -1552,14 +1567,6 @@ static void ena_napi_enable_all(struct ena_adapter *adapter) napi_enable(&adapter->ena_napi[i].napi); } -static void ena_restore_ethtool_params(struct ena_adapter *adapter) -{ - adapter->tx_usecs = 0; - adapter->rx_usecs = 0; - adapter->tx_frames = 1; - adapter->rx_frames = 1; -} - /* Configure the Rx forwarding */ static int ena_rss_configure(struct ena_adapter *adapter) { @@ -1609,8 +1616,6 @@ static int ena_up_complete(struct ena_adapter *adapter) /* enable transmits */ netif_tx_start_all_queues(adapter->netdev); - ena_restore_ethtool_params(adapter); - ena_napi_enable_all(adapter); return 0; @@ -1740,13 +1745,16 @@ static int ena_create_all_io_rx_queues(struct ena_adapter *adapter) rc = ena_create_io_rx_queue(adapter, i); if (rc) goto create_err; + INIT_WORK(&adapter->ena_napi[i].dim.work, ena_dim_work); } return 0; create_err: - while (i--) + while (i--) { + cancel_work_sync(&adapter->ena_napi[i].dim.work); ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(i)); + } return rc; } @@ -2419,6 +2427,9 @@ static void ena_config_host_info(struct ena_com_dev *ena_dev, ("K"[0] << ENA_ADMIN_HOST_INFO_MODULE_TYPE_SHIFT); host_info->num_cpus = num_online_cpus(); + host_info->driver_supported_features = + ENA_ADMIN_HOST_INFO_INTERRUPT_MODERATION_MASK; + rc = ena_com_set_host_attributes(ena_dev); if (rc) { if (rc == -EOPNOTSUPP) @@ -3485,10 +3496,12 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent) calc_queue_ctx.get_feat_ctx = &get_feat_ctx; calc_queue_ctx.pdev = pdev; - /* initial Tx interrupt delay, Assumes 1 usec granularity. + /* Initial Tx and RX interrupt delay. Assumes 1 usec granularity. * Updated during device initialization with the real granularity */ ena_dev->intr_moder_tx_interval = ENA_INTR_INITIAL_TX_INTERVAL_USECS; + ena_dev->intr_moder_rx_interval = ENA_INTR_INITIAL_RX_INTERVAL_USECS; + ena_dev->intr_delay_resolution = ENA_DEFAULT_INTR_DELAY_RESOLUTION; io_queue_num = ena_calc_io_queue_num(pdev, ena_dev, &get_feat_ctx); rc = ena_calc_queue_size(&calc_queue_ctx); if (rc || io_queue_num <= 0) { @@ -3618,7 +3631,6 @@ err_free_msix: ena_free_mgmnt_irq(adapter); ena_disable_msix(adapter); err_worker_destroy: - ena_com_destroy_interrupt_moderation(ena_dev); del_timer(&adapter->timer_service); err_netdev_destroy: free_netdev(netdev); @@ -3679,8 +3691,6 @@ static void ena_remove(struct pci_dev *pdev) pci_disable_device(pdev); - ena_com_destroy_interrupt_moderation(ena_dev); - vfree(ena_dev); } diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.h b/drivers/net/ethernet/amazon/ena/ena_netdev.h index efbcffd22215..72ee51a82ec7 100644 --- a/drivers/net/ethernet/amazon/ena/ena_netdev.h +++ b/drivers/net/ethernet/amazon/ena/ena_netdev.h @@ -34,6 +34,7 @@ #define ENA_H #include <linux/bitops.h> +#include <linux/dim.h> #include <linux/etherdevice.h> #include <linux/inetdevice.h> #include <linux/interrupt.h> @@ -153,6 +154,7 @@ struct ena_napi { struct ena_ring *tx_ring; struct ena_ring *rx_ring; u32 qid; + struct dim dim; }; struct ena_calc_queue_size_ctx { @@ -278,8 +280,7 @@ struct ena_ring { struct ena_com_rx_buf_info ena_bufs[ENA_PKT_MAX_BUFS]; u32 smoothed_interval; u32 per_napi_packets; - u32 per_napi_bytes; - enum ena_intr_moder_level moder_tbl_idx; + u16 non_empty_napi_events; struct u64_stats_sync syncp; union { struct ena_stats_tx tx_stats; @@ -329,9 +330,6 @@ struct ena_adapter { u32 missing_tx_completion_threshold; - u32 tx_usecs, rx_usecs; /* interrupt moderation */ - u32 tx_frames, rx_frames; /* interrupt moderation */ - u32 requested_tx_ring_size; u32 requested_rx_ring_size; |