diff options
Diffstat (limited to 'drivers/net/ethernet/mellanox/mlx5/core/en_tc.c')
-rw-r--r-- | drivers/net/ethernet/mellanox/mlx5/core/en_tc.c | 292 |
1 files changed, 108 insertions, 184 deletions
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index 7fc84f58e28a..b366c46a5604 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -63,6 +63,7 @@ #include "en/tc_tun.h" #include "en/mapping.h" #include "en/tc_ct.h" +#include "en/mod_hdr.h" #include "lib/devcom.h" #include "lib/geneve.h" #include "diag/en_tc_tracepoint.h" @@ -140,8 +141,7 @@ struct mlx5e_tc_flow { */ struct encap_flow_item encaps[MLX5_MAX_FLOW_FWD_VPORTS]; struct mlx5e_tc_flow *peer_flow; - struct mlx5e_mod_hdr_entry *mh; /* attached mod header instance */ - struct list_head mod_hdr; /* flows sharing the same mod hdr ID */ + struct mlx5e_mod_hdr_handle *mh; /* attached mod header instance */ struct mlx5e_hairpin_entry *hpe; /* attached hairpin instance */ struct list_head hairpin; /* flows sharing the same hairpin */ struct list_head peer; /* flows with peer flow */ @@ -180,17 +180,17 @@ struct mlx5e_tc_attr_to_reg_mapping mlx5e_tc_attr_to_reg_mappings[] = { }, [TUNNEL_TO_REG] = { .mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_1, - .moffset = 3, - .mlen = 1, + .moffset = 1, + .mlen = 3, .soffset = MLX5_BYTE_OFF(fte_match_param, misc_parameters_2.metadata_reg_c_1), }, [ZONE_TO_REG] = zone_to_reg_ct, + [ZONE_RESTORE_TO_REG] = zone_restore_to_reg_ct, [CTSTATE_TO_REG] = ctstate_to_reg_ct, [MARK_TO_REG] = mark_to_reg_ct, [LABELS_TO_REG] = labels_to_reg_ct, [FTEID_TO_REG] = fteid_to_reg_ct, - [TUPLEID_TO_REG] = tupleid_to_reg_ct, }; static void mlx5e_put_flow_tunnel_id(struct mlx5e_tc_flow *flow); @@ -219,6 +219,28 @@ mlx5e_tc_match_to_reg_match(struct mlx5_flow_spec *spec, spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; } +void +mlx5e_tc_match_to_reg_get_match(struct mlx5_flow_spec *spec, + enum mlx5e_tc_attr_to_reg type, + u32 *data, + u32 *mask) +{ + int soffset = mlx5e_tc_attr_to_reg_mappings[type].soffset; + int match_len = mlx5e_tc_attr_to_reg_mappings[type].mlen; + void *headers_c = spec->match_criteria; + void *headers_v = spec->match_value; + void *fmask, *fval; + + fmask = headers_c + soffset; + fval = headers_v + soffset; + + memcpy(mask, fmask, match_len); + memcpy(data, fval, match_len); + + *mask = be32_to_cpu((__force __be32)(*mask << (32 - (match_len * 8)))); + *data = be32_to_cpu((__force __be32)(*data << (32 - (match_len * 8)))); +} + int mlx5e_tc_match_to_reg_set(struct mlx5_core_dev *mdev, struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts, @@ -287,29 +309,6 @@ struct mlx5e_hairpin_entry { struct completion res_ready; }; -struct mod_hdr_key { - int num_actions; - void *actions; -}; - -struct mlx5e_mod_hdr_entry { - /* a node of a hash table which keeps all the mod_hdr entries */ - struct hlist_node mod_hdr_hlist; - - /* protects flows list */ - spinlock_t flows_lock; - /* flows sharing the same mod_hdr entry */ - struct list_head flows; - - struct mod_hdr_key key; - - struct mlx5_modify_hdr *modify_hdr; - - refcount_t refcnt; - struct completion res_ready; - int compl_result; -}; - static void mlx5e_tc_del_flow(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow); @@ -386,148 +385,43 @@ static bool mlx5e_is_offloaded_flow(struct mlx5e_tc_flow *flow) return flow_flag_test(flow, OFFLOADED); } -static inline u32 hash_mod_hdr_info(struct mod_hdr_key *key) -{ - return jhash(key->actions, - key->num_actions * MLX5_MH_ACT_SZ, 0); -} - -static inline int cmp_mod_hdr_info(struct mod_hdr_key *a, - struct mod_hdr_key *b) +static int get_flow_name_space(struct mlx5e_tc_flow *flow) { - if (a->num_actions != b->num_actions) - return 1; - - return memcmp(a->actions, b->actions, a->num_actions * MLX5_MH_ACT_SZ); + return mlx5e_is_eswitch_flow(flow) ? + MLX5_FLOW_NAMESPACE_FDB : MLX5_FLOW_NAMESPACE_KERNEL; } static struct mod_hdr_tbl * -get_mod_hdr_table(struct mlx5e_priv *priv, int namespace) +get_mod_hdr_table(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow) { struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; - return namespace == MLX5_FLOW_NAMESPACE_FDB ? &esw->offloads.mod_hdr : + return get_flow_name_space(flow) == MLX5_FLOW_NAMESPACE_FDB ? + &esw->offloads.mod_hdr : &priv->fs.tc.mod_hdr; } -static struct mlx5e_mod_hdr_entry * -mlx5e_mod_hdr_get(struct mod_hdr_tbl *tbl, struct mod_hdr_key *key, u32 hash_key) -{ - struct mlx5e_mod_hdr_entry *mh, *found = NULL; - - hash_for_each_possible(tbl->hlist, mh, mod_hdr_hlist, hash_key) { - if (!cmp_mod_hdr_info(&mh->key, key)) { - refcount_inc(&mh->refcnt); - found = mh; - break; - } - } - - return found; -} - -static void mlx5e_mod_hdr_put(struct mlx5e_priv *priv, - struct mlx5e_mod_hdr_entry *mh, - int namespace) -{ - struct mod_hdr_tbl *tbl = get_mod_hdr_table(priv, namespace); - - if (!refcount_dec_and_mutex_lock(&mh->refcnt, &tbl->lock)) - return; - hash_del(&mh->mod_hdr_hlist); - mutex_unlock(&tbl->lock); - - WARN_ON(!list_empty(&mh->flows)); - if (mh->compl_result > 0) - mlx5_modify_header_dealloc(priv->mdev, mh->modify_hdr); - - kfree(mh); -} - -static int get_flow_name_space(struct mlx5e_tc_flow *flow) -{ - return mlx5e_is_eswitch_flow(flow) ? - MLX5_FLOW_NAMESPACE_FDB : MLX5_FLOW_NAMESPACE_KERNEL; -} static int mlx5e_attach_mod_hdr(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow, struct mlx5e_tc_flow_parse_attr *parse_attr) { - int num_actions, actions_size, namespace, err; - struct mlx5e_mod_hdr_entry *mh; - struct mod_hdr_tbl *tbl; - struct mod_hdr_key key; - u32 hash_key; - - num_actions = parse_attr->mod_hdr_acts.num_actions; - actions_size = MLX5_MH_ACT_SZ * num_actions; - - key.actions = parse_attr->mod_hdr_acts.actions; - key.num_actions = num_actions; - - hash_key = hash_mod_hdr_info(&key); - - namespace = get_flow_name_space(flow); - tbl = get_mod_hdr_table(priv, namespace); - - mutex_lock(&tbl->lock); - mh = mlx5e_mod_hdr_get(tbl, &key, hash_key); - if (mh) { - mutex_unlock(&tbl->lock); - wait_for_completion(&mh->res_ready); - - if (mh->compl_result < 0) { - err = -EREMOTEIO; - goto attach_header_err; - } - goto attach_flow; - } - - mh = kzalloc(sizeof(*mh) + actions_size, GFP_KERNEL); - if (!mh) { - mutex_unlock(&tbl->lock); - return -ENOMEM; - } - - mh->key.actions = (void *)mh + sizeof(*mh); - memcpy(mh->key.actions, key.actions, actions_size); - mh->key.num_actions = num_actions; - spin_lock_init(&mh->flows_lock); - INIT_LIST_HEAD(&mh->flows); - refcount_set(&mh->refcnt, 1); - init_completion(&mh->res_ready); - - hash_add(tbl->hlist, &mh->mod_hdr_hlist, hash_key); - mutex_unlock(&tbl->lock); + struct mlx5_modify_hdr *modify_hdr; + struct mlx5e_mod_hdr_handle *mh; - mh->modify_hdr = mlx5_modify_header_alloc(priv->mdev, namespace, - mh->key.num_actions, - mh->key.actions); - if (IS_ERR(mh->modify_hdr)) { - err = PTR_ERR(mh->modify_hdr); - mh->compl_result = err; - goto alloc_header_err; - } - mh->compl_result = 1; - complete_all(&mh->res_ready); + mh = mlx5e_mod_hdr_attach(priv->mdev, get_mod_hdr_table(priv, flow), + get_flow_name_space(flow), + &parse_attr->mod_hdr_acts); + if (IS_ERR(mh)) + return PTR_ERR(mh); -attach_flow: - flow->mh = mh; - spin_lock(&mh->flows_lock); - list_add(&flow->mod_hdr, &mh->flows); - spin_unlock(&mh->flows_lock); + modify_hdr = mlx5e_mod_hdr_get(mh); if (mlx5e_is_eswitch_flow(flow)) - flow->esw_attr->modify_hdr = mh->modify_hdr; + flow->esw_attr->modify_hdr = modify_hdr; else - flow->nic_attr->modify_hdr = mh->modify_hdr; + flow->nic_attr->modify_hdr = modify_hdr; + flow->mh = mh; return 0; - -alloc_header_err: - complete_all(&mh->res_ready); -attach_header_err: - mlx5e_mod_hdr_put(priv, mh, namespace); - return err; } static void mlx5e_detach_mod_hdr(struct mlx5e_priv *priv, @@ -537,11 +431,8 @@ static void mlx5e_detach_mod_hdr(struct mlx5e_priv *priv, if (!flow->mh) return; - spin_lock(&flow->mh->flows_lock); - list_del(&flow->mod_hdr); - spin_unlock(&flow->mh->flows_lock); - - mlx5e_mod_hdr_put(priv, flow->mh, get_flow_name_space(flow)); + mlx5e_mod_hdr_detach(priv->mdev, get_mod_hdr_table(priv, flow), + flow->mh); flow->mh = NULL; } @@ -3086,6 +2977,7 @@ struct ipv6_hoplimit_word { static int is_action_keys_supported(const struct flow_action_entry *act, bool ct_flow, bool *modify_ip_header, + bool *modify_tuple, struct netlink_ext_ack *extack) { u32 mask, offset; @@ -3108,7 +3000,10 @@ static int is_action_keys_supported(const struct flow_action_entry *act, *modify_ip_header = true; } - if (ct_flow && offset >= offsetof(struct iphdr, saddr)) { + if (offset >= offsetof(struct iphdr, saddr)) + *modify_tuple = true; + + if (ct_flow && *modify_tuple) { NL_SET_ERR_MSG_MOD(extack, "can't offload re-write of ipv4 address with action ct"); return -EOPNOTSUPP; @@ -3123,28 +3018,36 @@ static int is_action_keys_supported(const struct flow_action_entry *act, *modify_ip_header = true; } - if (ct_flow && offset >= offsetof(struct ipv6hdr, saddr)) { + if (ct_flow && offset >= offsetof(struct ipv6hdr, saddr)) + *modify_tuple = true; + + if (ct_flow && *modify_tuple) { NL_SET_ERR_MSG_MOD(extack, "can't offload re-write of ipv6 address with action ct"); return -EOPNOTSUPP; } - } else if (ct_flow && (htype == FLOW_ACT_MANGLE_HDR_TYPE_TCP || - htype == FLOW_ACT_MANGLE_HDR_TYPE_UDP)) { - NL_SET_ERR_MSG_MOD(extack, - "can't offload re-write of transport header ports with action ct"); - return -EOPNOTSUPP; + } else if (htype == FLOW_ACT_MANGLE_HDR_TYPE_TCP || + htype == FLOW_ACT_MANGLE_HDR_TYPE_UDP) { + *modify_tuple = true; + if (ct_flow) { + NL_SET_ERR_MSG_MOD(extack, + "can't offload re-write of transport header ports with action ct"); + return -EOPNOTSUPP; + } } return 0; } -static bool modify_header_match_supported(struct mlx5_flow_spec *spec, +static bool modify_header_match_supported(struct mlx5e_priv *priv, + struct mlx5_flow_spec *spec, struct flow_action *flow_action, u32 actions, bool ct_flow, + bool ct_clear, struct netlink_ext_ack *extack) { const struct flow_action_entry *act; - bool modify_ip_header; + bool modify_ip_header, modify_tuple; void *headers_c; void *headers_v; u16 ethertype; @@ -3161,23 +3064,39 @@ static bool modify_header_match_supported(struct mlx5_flow_spec *spec, goto out_ok; modify_ip_header = false; + modify_tuple = false; flow_action_for_each(i, act, flow_action) { if (act->id != FLOW_ACTION_MANGLE && act->id != FLOW_ACTION_ADD) continue; err = is_action_keys_supported(act, ct_flow, - &modify_ip_header, extack); + &modify_ip_header, + &modify_tuple, extack); if (err) return err; } + /* Add ct_state=-trk match so it will be offloaded for non ct flows + * (or after clear action), as otherwise, since the tuple is changed, + * we can't restore ct state + */ + if (!ct_clear && modify_tuple && + mlx5_tc_ct_add_no_trk_match(priv, spec)) { + NL_SET_ERR_MSG_MOD(extack, + "can't offload tuple modify header with ct matches"); + netdev_info(priv->netdev, + "can't offload tuple modify header with ct matches"); + return false; + } + ip_proto = MLX5_GET(fte_match_set_lyr_2_4, headers_v, ip_protocol); if (modify_ip_header && ip_proto != IPPROTO_TCP && ip_proto != IPPROTO_UDP && ip_proto != IPPROTO_ICMP) { NL_SET_ERR_MSG_MOD(extack, "can't offload re-write of non TCP/UDP"); - pr_info("can't offload re-write of ip proto %d\n", ip_proto); + netdev_info(priv->netdev, "can't offload re-write of ip proto %d\n", + ip_proto); return false; } @@ -3191,13 +3110,14 @@ static bool actions_match_supported(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow, struct netlink_ext_ack *extack) { - bool ct_flow; + bool ct_flow = false, ct_clear = false; u32 actions; - ct_flow = flow_flag_test(flow, CT); if (mlx5e_is_eswitch_flow(flow)) { actions = flow->esw_attr->action; - + ct_clear = flow->esw_attr->ct_attr.ct_action & + TCA_CT_ACT_CLEAR; + ct_flow = flow_flag_test(flow, CT) && !ct_clear; if (flow->esw_attr->split_count && ct_flow) { /* All registers used by ct are cleared when using * split rules. @@ -3211,9 +3131,10 @@ static bool actions_match_supported(struct mlx5e_priv *priv, } if (actions & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) - return modify_header_match_supported(&parse_attr->spec, + return modify_header_match_supported(priv, &parse_attr->spec, flow_action, actions, - ct_flow, extack); + ct_flow, ct_clear, + extack); return true; } @@ -4408,7 +4329,6 @@ mlx5e_alloc_flow(struct mlx5e_priv *priv, int attr_size, flow->priv = priv; for (out_index = 0; out_index < MLX5_MAX_FLOW_FWD_VPORTS; out_index++) INIT_LIST_HEAD(&flow->encaps[out_index].list); - INIT_LIST_HEAD(&flow->mod_hdr); INIT_LIST_HEAD(&flow->hairpin); INIT_LIST_HEAD(&flow->l3_to_l2_reformat); refcount_set(&flow->refcnt, 1); @@ -4480,11 +4400,12 @@ __mlx5e_add_fdb_flow(struct mlx5e_priv *priv, if (err) goto err_free; - err = parse_tc_fdb_actions(priv, &rule->action, flow, extack, filter_dev); + /* actions validation depends on parsing the ct matches first */ + err = mlx5_tc_ct_parse_match(priv, &parse_attr->spec, f, extack); if (err) goto err_free; - err = mlx5_tc_ct_parse_match(priv, &parse_attr->spec, f, extack); + err = parse_tc_fdb_actions(priv, &rule->action, flow, extack, filter_dev); if (err) goto err_free; @@ -4670,9 +4591,10 @@ static bool is_flow_rule_duplicate_allowed(struct net_device *dev, struct mlx5e_rep_priv *rpriv) { /* Offloaded flow rule is allowed to duplicate on non-uplink representor - * sharing tc block with other slaves of a lag device. + * sharing tc block with other slaves of a lag device. Rpriv can be NULL if this + * function is called from NIC mode. */ - return netif_is_lag_port(dev) && rpriv->rep->vport != MLX5_VPORT_UPLINK; + return netif_is_lag_port(dev) && rpriv && rpriv->rep->vport != MLX5_VPORT_UPLINK; } int mlx5e_configure_flower(struct net_device *dev, struct mlx5e_priv *priv, @@ -4686,13 +4608,12 @@ int mlx5e_configure_flower(struct net_device *dev, struct mlx5e_priv *priv, rcu_read_lock(); flow = rhashtable_lookup(tc_ht, &f->cookie, tc_ht_params); - rcu_read_unlock(); if (flow) { /* Same flow rule offloaded to non-uplink representor sharing tc block, * just return 0. */ if (is_flow_rule_duplicate_allowed(dev, rpriv) && flow->orig_dev != dev) - goto out; + goto rcu_unlock; NL_SET_ERR_MSG_MOD(extack, "flow cookie already exists, ignoring"); @@ -4700,8 +4621,12 @@ int mlx5e_configure_flower(struct net_device *dev, struct mlx5e_priv *priv, "flow cookie %lx already exists, ignoring\n", f->cookie); err = -EEXIST; - goto out; + goto rcu_unlock; } +rcu_unlock: + rcu_read_unlock(); + if (flow) + goto out; trace_mlx5e_configure_flower(f); err = mlx5e_tc_add_flow(priv, f, flags, dev, &flow); @@ -4828,7 +4753,7 @@ int mlx5e_stats_flower(struct net_device *dev, struct mlx5e_priv *priv, no_peer_counter: mlx5_devcom_release_peer_data(devcom, MLX5_DEVCOM_ESW_OFFLOADS); out: - flow_stats_update(&f->stats, bytes, packets, lastuse, + flow_stats_update(&f->stats, bytes, packets, 0, lastuse, FLOW_ACTION_HW_STATS_DELAYED); trace_mlx5e_stats_flower(f); errout: @@ -4946,7 +4871,7 @@ void mlx5e_tc_stats_matchall(struct mlx5e_priv *priv, dpkts = cur_stats.rx_packets - rpriv->prev_vf_vport_stats.rx_packets; dbytes = cur_stats.rx_bytes - rpriv->prev_vf_vport_stats.rx_bytes; rpriv->prev_vf_vport_stats = cur_stats; - flow_stats_update(&ma->stats, dbytes, dpkts, jiffies, + flow_stats_update(&ma->stats, dbytes, dpkts, 0, jiffies, FLOW_ACTION_HW_STATS_DELAYED); } @@ -5011,9 +4936,8 @@ int mlx5e_tc_nic_init(struct mlx5e_priv *priv) struct mlx5e_tc_table *tc = &priv->fs.tc; int err; + mlx5e_mod_hdr_tbl_init(&tc->mod_hdr); mutex_init(&tc->t_lock); - mutex_init(&tc->mod_hdr.lock); - hash_init(tc->mod_hdr.hlist); mutex_init(&tc->hairpin_tbl_lock); hash_init(tc->hairpin_tbl); @@ -5051,7 +4975,7 @@ void mlx5e_tc_nic_cleanup(struct mlx5e_priv *priv) &tc->netdevice_nb, &tc->netdevice_nn); - mutex_destroy(&tc->mod_hdr.lock); + mlx5e_mod_hdr_tbl_destroy(&tc->mod_hdr); mutex_destroy(&tc->hairpin_tbl_lock); rhashtable_destroy(&tc->ht); |