1 files changed, 210 insertions, 84 deletions

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/sample.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/sample.c
index 739292d52aca..6552ecee3f9b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/sample.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/sample.c
@@ -4,6 +4,7 @@
 #include <linux/skbuff.h>
 #include <net/psample.h>
 #include "en/mapping.h"
+#include "en/tc/post_act.h"
 #include "sample.h"
 #include "eswitch.h"
 #include "en_tc.h"
@@ -25,6 +26,7 @@ struct mlx5e_tc_psample {
 	struct mutex ht_lock; /* protect hashtbl */
 	DECLARE_HASHTABLE(restore_hashtbl, 8);
 	struct mutex restore_lock; /* protect restore_hashtbl */
+	struct mlx5e_post_act *post_act;
 };
 
 struct mlx5e_sampler {
@@ -41,13 +43,16 @@ struct mlx5e_sample_flow {
 	struct mlx5e_sample_restore *restore;
 	struct mlx5_flow_attr *pre_attr;
 	struct mlx5_flow_handle *pre_rule;
-	struct mlx5_flow_handle *rule;
+	struct mlx5_flow_attr *post_attr;
+	struct mlx5_flow_handle *post_rule;
+	struct mlx5e_post_act_handle *post_act_handle;
 };
 
 struct mlx5e_sample_restore {
 	struct hlist_node hlist;
 	struct mlx5_modify_hdr *modify_hdr;
 	struct mlx5_flow_handle *rule;
+	struct mlx5e_post_act_handle *post_act_handle;
 	u32 obj_id;
 	int count;
 };
@@ -217,8 +222,15 @@ sampler_put(struct mlx5e_tc_psample *tc_psample, struct mlx5e_sampler *sampler)
 	mutex_unlock(&tc_psample->ht_lock);
 }
 
+/* obj_id is used to restore the sample parameters.
+ * Set fte_id in original flow table, then match it in the default table.
+ * Only set it for NICs can preserve reg_c or decap action. For other cases,
+ * use the same match in the default table.
+ * Use one header rewrite for both obj_id and fte_id.
+ */
 static struct mlx5_modify_hdr *
-sample_metadata_rule_get(struct mlx5_core_dev *mdev, u32 obj_id)
+sample_modify_hdr_get(struct mlx5_core_dev *mdev, u32 obj_id,
+		      struct mlx5e_post_act_handle *handle)
 {
 	struct mlx5e_tc_mod_hdr_acts mod_acts = {};
 	struct mlx5_modify_hdr *modify_hdr;
@@ -229,6 +241,12 @@ sample_metadata_rule_get(struct mlx5_core_dev *mdev, u32 obj_id)
 	if (err)
 		goto err_set_regc0;
 
+	if (handle) {
+		err = mlx5e_tc_post_act_set_handle(mdev, handle, &mod_acts);
+		if (err)
+			goto err_post_act;
+	}
+
 	modify_hdr = mlx5_modify_header_alloc(mdev, MLX5_FLOW_NAMESPACE_FDB,
 					      mod_acts.num_actions,
 					      mod_acts.actions);
@@ -241,23 +259,40 @@ sample_metadata_rule_get(struct mlx5_core_dev *mdev, u32 obj_id)
 	return modify_hdr;
 
 err_modify_hdr:
+err_post_act:
 	dealloc_mod_hdr_actions(&mod_acts);
 err_set_regc0:
 	return ERR_PTR(err);
 }
 
+static u32
+restore_hash(u32 obj_id, struct mlx5e_post_act_handle *post_act_handle)
+{
+	return jhash_2words(obj_id, hash32_ptr(post_act_handle), 0);
+}
+
+static bool
+restore_equal(struct mlx5e_sample_restore *restore, u32 obj_id,
+	      struct mlx5e_post_act_handle *post_act_handle)
+{
+	return restore->obj_id == obj_id && restore->post_act_handle == post_act_handle;
+}
+
 static struct mlx5e_sample_restore *
-sample_restore_get(struct mlx5e_tc_psample *tc_psample, u32 obj_id)
+sample_restore_get(struct mlx5e_tc_psample *tc_psample, u32 obj_id,
+		   struct mlx5e_post_act_handle *post_act_handle)
 {
 	struct mlx5_eswitch *esw = tc_psample->esw;
 	struct mlx5_core_dev *mdev = esw->dev;
 	struct mlx5e_sample_restore *restore;
 	struct mlx5_modify_hdr *modify_hdr;
+	u32 hash_key;
 	int err;
 
 	mutex_lock(&tc_psample->restore_lock);
-	hash_for_each_possible(tc_psample->restore_hashtbl, restore, hlist, obj_id)
-		if (restore->obj_id == obj_id)
+	hash_key = restore_hash(obj_id, post_act_handle);
+	hash_for_each_possible(tc_psample->restore_hashtbl, restore, hlist, hash_key)
+		if (restore_equal(restore, obj_id, post_act_handle))
 			goto add_ref;
 
 	restore = kzalloc(sizeof(*restore), GFP_KERNEL);
@@ -266,8 +301,9 @@ sample_restore_get(struct mlx5e_tc_psample *tc_psample, u32 obj_id)
 		goto err_alloc;
 	}
 	restore->obj_id = obj_id;
+	restore->post_act_handle = post_act_handle;
 
-	modify_hdr = sample_metadata_rule_get(mdev, obj_id);
+	modify_hdr = sample_modify_hdr_get(mdev, obj_id, post_act_handle);
 	if (IS_ERR(modify_hdr)) {
 		err = PTR_ERR(modify_hdr);
 		goto err_modify_hdr;
@@ -280,7 +316,7 @@ sample_restore_get(struct mlx5e_tc_psample *tc_psample, u32 obj_id)
 		goto err_restore;
 	}
 
-	hash_add(tc_psample->restore_hashtbl, &restore->hlist, obj_id);
+	hash_add(tc_psample->restore_hashtbl, &restore->hlist, hash_key);
 add_ref:
 	restore->count++;
 	mutex_unlock(&tc_psample->restore_lock);
@@ -325,6 +361,87 @@ void mlx5e_tc_sample_skb(struct sk_buff *skb, struct mlx5_mapped_obj *mapped_obj
 	psample_sample_packet(&psample_group, skb, mapped_obj->sample.rate, &md);
 }
 
+static int
+add_post_rule(struct mlx5_eswitch *esw, struct mlx5e_sample_flow *sample_flow,
+	      struct mlx5_flow_spec *spec, struct mlx5_flow_attr *attr,
+	      u32 *default_tbl_id)
+{
+	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
+	u32 attr_sz = ns_to_attr_sz(MLX5_FLOW_NAMESPACE_FDB);
+	struct mlx5_vport_tbl_attr per_vport_tbl_attr;
+	struct mlx5_flow_table *default_tbl;
+	struct mlx5_flow_attr *post_attr;
+	int err;
+
+	/* Allocate default table per vport, chain and prio. Otherwise, there is
+	 * only one default table for the same sampler object. Rules with different
+	 * prio and chain may overlap. For CT sample action, per vport default
+	 * table is needed to resotre the metadata.
+	 */
+	per_vport_tbl_attr.chain = attr->chain;
+	per_vport_tbl_attr.prio = attr->prio;
+	per_vport_tbl_attr.vport = esw_attr->in_rep->vport;
+	per_vport_tbl_attr.vport_ns = &mlx5_esw_vport_tbl_sample_ns;
+	default_tbl = mlx5_esw_vporttbl_get(esw, &per_vport_tbl_attr);
+	if (IS_ERR(default_tbl)) {
+		err = PTR_ERR(default_tbl);
+		goto err_default_tbl;
+	}
+	*default_tbl_id = default_tbl->id;
+
+	post_attr = mlx5_alloc_flow_attr(MLX5_FLOW_NAMESPACE_FDB);
+	if (!post_attr) {
+		err = -ENOMEM;
+		goto err_attr;
+	}
+	sample_flow->post_attr = post_attr;
+	memcpy(post_attr, attr, attr_sz);
+	/* Perform the original matches on the default table.
+	 * Offload all actions except the sample action.
+	 */
+	post_attr->chain = 0;
+	post_attr->prio = 0;
+	post_attr->ft = default_tbl;
+	post_attr->flags = MLX5_ESW_ATTR_FLAG_NO_IN_PORT;
+
+	/* When offloading sample and encap action, if there is no valid
+	 * neigh data struct, a slow path rule is offloaded first. Source
+	 * port metadata match is set at that time. A per vport table is
+	 * already allocated. No need to match it again. So clear the source
+	 * port metadata match.
+	 */
+	mlx5_eswitch_clear_rule_source_port(esw, spec);
+	sample_flow->post_rule = mlx5_eswitch_add_offloaded_rule(esw, spec, post_attr);
+	if (IS_ERR(sample_flow->post_rule)) {
+		err = PTR_ERR(sample_flow->post_rule);
+		goto err_rule;
+	}
+	return 0;
+
+err_rule:
+	kfree(post_attr);
+err_attr:
+	mlx5_esw_vporttbl_put(esw, &per_vport_tbl_attr);
+err_default_tbl:
+	return err;
+}
+
+static void
+del_post_rule(struct mlx5_eswitch *esw, struct mlx5e_sample_flow *sample_flow,
+	      struct mlx5_flow_attr *attr)
+{
+	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
+	struct mlx5_vport_tbl_attr tbl_attr;
+
+	mlx5_eswitch_del_offloaded_rule(esw, sample_flow->post_rule, sample_flow->post_attr);
+	kfree(sample_flow->post_attr);
+	tbl_attr.chain = attr->chain;
+	tbl_attr.prio = attr->prio;
+	tbl_attr.vport = esw_attr->in_rep->vport;
+	tbl_attr.vport_ns = &mlx5_esw_vport_tbl_sample_ns;
+	mlx5_esw_vporttbl_put(esw, &tbl_attr);
+}
+
 /* For the following typical flow table:
  *
  * +-------------------------------+
@@ -342,8 +459,9 @@ void mlx5e_tc_sample_skb(struct sk_buff *skb, struct mlx5_mapped_obj *mapped_obj
  *         +---------------------+
  *         +   original match    +
  *         +---------------------+
- *                    |
- *                    v
+ *               | set fte_id (if reg_c preserve cap)
+ *               | do decap (if required)
+ *               v
  * +------------------------------------------------+
  * +                Flow Sampler Object             +
  * +------------------------------------------------+
@@ -353,13 +471,22 @@ void mlx5e_tc_sample_skb(struct sk_buff *skb, struct mlx5_mapped_obj *mapped_obj
  * +------------------------------------------------+
  *            |                            |
  *            v                            v
- * +-----------------------------+  +----------------------------------------+
- * +        sample table         +  + default table per <vport, chain, prio> +
- * +-----------------------------+  +----------------------------------------+
- * + forward to management vport +  +            original match              +
- * +-----------------------------+  +----------------------------------------+
- *                                  +            other actions               +
- *                                  +----------------------------------------+
+ * +-----------------------------+  +-------------------+
+ * +        sample table         +  +   default table   +
+ * +-----------------------------+  +-------------------+
+ * + forward to management vport +             |
+ * +-----------------------------+             |
+ *                                     +-------+------+
+ *                                     |              |reg_c preserve cap
+ *                                     |              |or decap action
+ *                                     v              v
+ *                        +-----------------+   +-------------+
+ *                        + per vport table +   + post action +
+ *                        +-----------------+   +-------------+
+ *                        + original match  +
+ *                        +-----------------+
+ *                        + other actions   +
+ *                        +-----------------+
  */
 struct mlx5_flow_handle *
 mlx5e_tc_sample_offload(struct mlx5e_tc_psample *tc_psample,
@@ -367,15 +494,15 @@ mlx5e_tc_sample_offload(struct mlx5e_tc_psample *tc_psample,
 			struct mlx5_flow_attr *attr,
 			u32 tunnel_id)
 {
+	struct mlx5e_post_act_handle *post_act_handle = NULL;
 	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
-	struct mlx5_vport_tbl_attr per_vport_tbl_attr;
 	struct mlx5_esw_flow_attr *pre_esw_attr;
 	struct mlx5_mapped_obj restore_obj = {};
 	struct mlx5e_sample_flow *sample_flow;
 	struct mlx5e_sample_attr *sample_attr;
-	struct mlx5_flow_table *default_tbl;
 	struct mlx5_flow_attr *pre_attr;
 	struct mlx5_eswitch *esw;
+	u32 default_tbl_id;
 	u32 obj_id;
 	int err;
 
@@ -395,40 +522,31 @@ mlx5e_tc_sample_offload(struct mlx5e_tc_psample *tc_psample,
 	sample_attr = attr->sample_attr;
 	sample_attr->sample_flow = sample_flow;
 
-	/* Allocate default table per vport, chain and prio. Otherwise, there is
-	 * only one default table for the same sampler object. Rules with different
-	 * prio and chain may overlap. For CT sample action, per vport default
-	 * table is needed to resotre the metadata.
-	 */
-	per_vport_tbl_attr.chain = attr->chain;
-	per_vport_tbl_attr.prio = attr->prio;
-	per_vport_tbl_attr.vport = esw_attr->in_rep->vport;
-	per_vport_tbl_attr.vport_ns = &mlx5_esw_vport_tbl_sample_ns;
-	default_tbl = mlx5_esw_vporttbl_get(esw, &per_vport_tbl_attr);
-	if (IS_ERR(default_tbl)) {
-		err = PTR_ERR(default_tbl);
-		goto err_default_tbl;
-	}
-
-	/* Perform the original matches on the default table.
-	 * Offload all actions except the sample action.
-	 */
-	sample_attr->sample_default_tbl = default_tbl;
-	/* When offloading sample and encap action, if there is no valid
-	 * neigh data struct, a slow path rule is offloaded first. Source
-	 * port metadata match is set at that time. A per vport table is
-	 * already allocated. No need to match it again. So clear the source
-	 * port metadata match.
+	/* For NICs with reg_c_preserve support or decap action, use
+	 * post action instead of the per vport, chain and prio table.
+	 * Only match the fte id instead of the same match in the
+	 * original flow table.
 	 */
-	mlx5_eswitch_clear_rule_source_port(esw, spec);
-	sample_flow->rule = mlx5_eswitch_add_offloaded_rule(esw, spec, attr);
-	if (IS_ERR(sample_flow->rule)) {
-		err = PTR_ERR(sample_flow->rule);
-		goto err_offload_rule;
+	if (MLX5_CAP_GEN(esw->dev, reg_c_preserve) ||
+	    attr->action & MLX5_FLOW_CONTEXT_ACTION_DECAP) {
+		struct mlx5_flow_table *ft;
+
+		ft = mlx5e_tc_post_act_get_ft(tc_psample->post_act);
+		default_tbl_id = ft->id;
+		post_act_handle = mlx5e_tc_post_act_add(tc_psample->post_act, attr);
+		if (IS_ERR(post_act_handle)) {
+			err = PTR_ERR(post_act_handle);
+			goto err_post_act;
+		}
+		sample_flow->post_act_handle = post_act_handle;
+	} else {
+		err = add_post_rule(esw, sample_flow, spec, attr, &default_tbl_id);
+		if (err)
+			goto err_post_rule;
 	}
 
 	/* Create sampler object. */
-	sample_flow->sampler = sampler_get(tc_psample, sample_attr->rate, default_tbl->id);
+	sample_flow->sampler = sampler_get(tc_psample, sample_attr->rate, default_tbl_id);
 	if (IS_ERR(sample_flow->sampler)) {
 		err = PTR_ERR(sample_flow->sampler);
 		goto err_sampler;
@@ -446,7 +564,7 @@ mlx5e_tc_sample_offload(struct mlx5e_tc_psample *tc_psample,
 	sample_attr->restore_obj_id = obj_id;
 
 	/* Create sample restore context. */
-	sample_flow->restore = sample_restore_get(tc_psample, obj_id);
+	sample_flow->restore = sample_restore_get(tc_psample, obj_id, post_act_handle);
 	if (IS_ERR(sample_flow->restore)) {
 		err = PTR_ERR(sample_flow->restore);
 		goto err_sample_restore;
@@ -458,19 +576,21 @@ mlx5e_tc_sample_offload(struct mlx5e_tc_psample *tc_psample,
 	pre_attr = mlx5_alloc_flow_attr(MLX5_FLOW_NAMESPACE_FDB);
 	if (!pre_attr) {
 		err = -ENOMEM;
-		goto err_alloc_flow_attr;
-	}
-	sample_attr = kzalloc(sizeof(*sample_attr), GFP_KERNEL);
-	if (!sample_attr) {
-		err = -ENOMEM;
-		goto err_alloc_sample_attr;
+		goto err_alloc_pre_flow_attr;
 	}
 	pre_attr->action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
+	/* For decap action, do decap in the original flow table instead of the
+	 * default flow table.
+	 */
+	if (tunnel_id)
+		pre_attr->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP;
 	pre_attr->modify_hdr = sample_flow->restore->modify_hdr;
 	pre_attr->flags = MLX5_ESW_ATTR_FLAG_SAMPLE;
+	pre_attr->inner_match_level = attr->inner_match_level;
+	pre_attr->outer_match_level = attr->outer_match_level;
 	pre_attr->chain = attr->chain;
 	pre_attr->prio = attr->prio;
-	pre_attr->sample_attr = sample_attr;
+	pre_attr->sample_attr = attr->sample_attr;
 	sample_attr->sampler_id = sample_flow->sampler->sampler_id;
 	pre_esw_attr = pre_attr->esw_attr;
 	pre_esw_attr->in_mdev = esw_attr->in_mdev;
@@ -482,28 +602,23 @@ mlx5e_tc_sample_offload(struct mlx5e_tc_psample *tc_psample,
 	}
 	sample_flow->pre_attr = pre_attr;
 
-	return sample_flow->rule;
+	return sample_flow->post_rule;
 
 err_pre_offload_rule:
-	kfree(sample_attr);
-err_alloc_sample_attr:
 	kfree(pre_attr);
-err_alloc_flow_attr:
+err_alloc_pre_flow_attr:
 	sample_restore_put(tc_psample, sample_flow->restore);
 err_sample_restore:
 	mapping_remove(esw->offloads.reg_c0_obj_pool, obj_id);
 err_obj_id:
 	sampler_put(tc_psample, sample_flow->sampler);
 err_sampler:
-	/* For sample offload, rule is added in default_tbl. No need to call
-	 * mlx5_esw_chains_put_table()
-	 */
-	attr->prio = 0;
-	attr->chain = 0;
-	mlx5_eswitch_del_offloaded_rule(esw, sample_flow->rule, attr);
-err_offload_rule:
-	mlx5_esw_vporttbl_put(esw, &per_vport_tbl_attr);
-err_default_tbl:
+	if (!post_act_handle)
+		del_post_rule(esw, sample_flow, attr);
+err_post_rule:
+	if (post_act_handle)
+		mlx5e_tc_post_act_del(tc_psample->post_act, post_act_handle);
+err_post_act:
 	kfree(sample_flow);
 	return ERR_PTR(err);
 }
@@ -516,7 +631,6 @@ mlx5e_tc_sample_unoffload(struct mlx5e_tc_psample *tc_psample,
 	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
 	struct mlx5e_sample_flow *sample_flow;
 	struct mlx5_vport_tbl_attr tbl_attr;
-	struct mlx5_flow_attr *pre_attr;
 	struct mlx5_eswitch *esw;
 
 	if (IS_ERR_OR_NULL(tc_psample))
@@ -531,28 +645,35 @@ mlx5e_tc_sample_unoffload(struct mlx5e_tc_psample *tc_psample,
 		return;
 	}
 
+	/* The following delete order can't be changed, otherwise,
+	 * will hit fw syndromes.
+	 */
 	sample_flow = attr->sample_attr->sample_flow;
-	pre_attr = sample_flow->pre_attr;
-	memset(pre_attr, 0, sizeof(*pre_attr));
-	mlx5_eswitch_del_offloaded_rule(esw, sample_flow->pre_rule, pre_attr);
-	mlx5_eswitch_del_offloaded_rule(esw, sample_flow->rule, attr);
+	mlx5_eswitch_del_offloaded_rule(esw, sample_flow->pre_rule, sample_flow->pre_attr);
+	if (!sample_flow->post_act_handle)
+		mlx5_eswitch_del_offloaded_rule(esw, sample_flow->post_rule,
+						sample_flow->post_attr);
 
 	sample_restore_put(tc_psample, sample_flow->restore);
 	mapping_remove(esw->offloads.reg_c0_obj_pool, attr->sample_attr->restore_obj_id);
 	sampler_put(tc_psample, sample_flow->sampler);
-	tbl_attr.chain = attr->chain;
-	tbl_attr.prio = attr->prio;
-	tbl_attr.vport = esw_attr->in_rep->vport;
-	tbl_attr.vport_ns = &mlx5_esw_vport_tbl_sample_ns;
-	mlx5_esw_vporttbl_put(esw, &tbl_attr);
+	if (sample_flow->post_act_handle) {
+		mlx5e_tc_post_act_del(tc_psample->post_act, sample_flow->post_act_handle);
+	} else {
+		tbl_attr.chain = attr->chain;
+		tbl_attr.prio = attr->prio;
+		tbl_attr.vport = esw_attr->in_rep->vport;
+		tbl_attr.vport_ns = &mlx5_esw_vport_tbl_sample_ns;
+		mlx5_esw_vporttbl_put(esw, &tbl_attr);
+		kfree(sample_flow->post_attr);
+	}
 
-	kfree(pre_attr->sample_attr);
-	kfree(pre_attr);
+	kfree(sample_flow->pre_attr);
 	kfree(sample_flow);
 }
 
 struct mlx5e_tc_psample *
-mlx5e_tc_sample_init(struct mlx5_eswitch *esw)
+mlx5e_tc_sample_init(struct mlx5_eswitch *esw, struct mlx5e_post_act *post_act)
 {
 	struct mlx5e_tc_psample *tc_psample;
 	int err;
@@ -560,17 +681,22 @@ mlx5e_tc_sample_init(struct mlx5_eswitch *esw)
 	tc_psample = kzalloc(sizeof(*tc_psample), GFP_KERNEL);
 	if (!tc_psample)
 		return ERR_PTR(-ENOMEM);
+	if (IS_ERR_OR_NULL(post_act)) {
+		err = PTR_ERR(post_act);
+		goto err_post_act;
+	}
+	tc_psample->post_act = post_act;
 	tc_psample->esw = esw;
 	err = sampler_termtbl_create(tc_psample);
 	if (err)
-		goto err_termtbl;
+		goto err_post_act;
 
 	mutex_init(&tc_psample->ht_lock);
 	mutex_init(&tc_psample->restore_lock);
 
 	return tc_psample;
 
-err_termtbl:
+err_post_act:
 	kfree(tc_psample);
 	return ERR_PTR(err);
 }