Contributors: 1
Author Tokens Token Proportion Commits Commit Proportion
Maor Gottlieb 3021 100.00% 2 100.00%
Total 3021 2 


// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
// Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES.

#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/tcp.h>
#include <linux/mlx5/fs.h>
#include <linux/mlx5/driver.h>
#include "mlx5_core.h"
#include "lib/fs_ttc.h"

#define MLX5_TTC_NUM_GROUPS	3
#define MLX5_TTC_GROUP1_SIZE	(BIT(3) + MLX5_NUM_TUNNEL_TT)
#define MLX5_TTC_GROUP2_SIZE	 BIT(1)
#define MLX5_TTC_GROUP3_SIZE	 BIT(0)
#define MLX5_TTC_TABLE_SIZE	(MLX5_TTC_GROUP1_SIZE +\
				 MLX5_TTC_GROUP2_SIZE +\
				 MLX5_TTC_GROUP3_SIZE)

#define MLX5_INNER_TTC_NUM_GROUPS	3
#define MLX5_INNER_TTC_GROUP1_SIZE	BIT(3)
#define MLX5_INNER_TTC_GROUP2_SIZE	BIT(1)
#define MLX5_INNER_TTC_GROUP3_SIZE	BIT(0)
#define MLX5_INNER_TTC_TABLE_SIZE	(MLX5_INNER_TTC_GROUP1_SIZE +\
					 MLX5_INNER_TTC_GROUP2_SIZE +\
					 MLX5_INNER_TTC_GROUP3_SIZE)

/* L3/L4 traffic type classifier */
struct mlx5_ttc_table {
	int num_groups;
	struct mlx5_flow_table *t;
	struct mlx5_flow_group **g;
	struct mlx5_ttc_rule rules[MLX5_NUM_TT];
	struct mlx5_flow_handle *tunnel_rules[MLX5_NUM_TUNNEL_TT];
};

struct mlx5_flow_table *mlx5_get_ttc_flow_table(struct mlx5_ttc_table *ttc)
{
	return ttc->t;
}

static void mlx5_cleanup_ttc_rules(struct mlx5_ttc_table *ttc)
{
	int i;

	for (i = 0; i < MLX5_NUM_TT; i++) {
		if (!IS_ERR_OR_NULL(ttc->rules[i].rule)) {
			mlx5_del_flow_rules(ttc->rules[i].rule);
			ttc->rules[i].rule = NULL;
		}
	}

	for (i = 0; i < MLX5_NUM_TUNNEL_TT; i++) {
		if (!IS_ERR_OR_NULL(ttc->tunnel_rules[i])) {
			mlx5_del_flow_rules(ttc->tunnel_rules[i]);
			ttc->tunnel_rules[i] = NULL;
		}
	}
}

struct mlx5_etype_proto {
	u16 etype;
	u8 proto;
};

static struct mlx5_etype_proto ttc_rules[] = {
	[MLX5_TT_IPV4_TCP] = {
		.etype = ETH_P_IP,
		.proto = IPPROTO_TCP,
	},
	[MLX5_TT_IPV6_TCP] = {
		.etype = ETH_P_IPV6,
		.proto = IPPROTO_TCP,
	},
	[MLX5_TT_IPV4_UDP] = {
		.etype = ETH_P_IP,
		.proto = IPPROTO_UDP,
	},
	[MLX5_TT_IPV6_UDP] = {
		.etype = ETH_P_IPV6,
		.proto = IPPROTO_UDP,
	},
	[MLX5_TT_IPV4_IPSEC_AH] = {
		.etype = ETH_P_IP,
		.proto = IPPROTO_AH,
	},
	[MLX5_TT_IPV6_IPSEC_AH] = {
		.etype = ETH_P_IPV6,
		.proto = IPPROTO_AH,
	},
	[MLX5_TT_IPV4_IPSEC_ESP] = {
		.etype = ETH_P_IP,
		.proto = IPPROTO_ESP,
	},
	[MLX5_TT_IPV6_IPSEC_ESP] = {
		.etype = ETH_P_IPV6,
		.proto = IPPROTO_ESP,
	},
	[MLX5_TT_IPV4] = {
		.etype = ETH_P_IP,
		.proto = 0,
	},
	[MLX5_TT_IPV6] = {
		.etype = ETH_P_IPV6,
		.proto = 0,
	},
	[MLX5_TT_ANY] = {
		.etype = 0,
		.proto = 0,
	},
};

static struct mlx5_etype_proto ttc_tunnel_rules[] = {
	[MLX5_TT_IPV4_GRE] = {
		.etype = ETH_P_IP,
		.proto = IPPROTO_GRE,
	},
	[MLX5_TT_IPV6_GRE] = {
		.etype = ETH_P_IPV6,
		.proto = IPPROTO_GRE,
	},
	[MLX5_TT_IPV4_IPIP] = {
		.etype = ETH_P_IP,
		.proto = IPPROTO_IPIP,
	},
	[MLX5_TT_IPV6_IPIP] = {
		.etype = ETH_P_IPV6,
		.proto = IPPROTO_IPIP,
	},
	[MLX5_TT_IPV4_IPV6] = {
		.etype = ETH_P_IP,
		.proto = IPPROTO_IPV6,
	},
	[MLX5_TT_IPV6_IPV6] = {
		.etype = ETH_P_IPV6,
		.proto = IPPROTO_IPV6,
	},

};

u8 mlx5_get_proto_by_tunnel_type(enum mlx5_tunnel_types tt)
{
	return ttc_tunnel_rules[tt].proto;
}

static bool mlx5_tunnel_proto_supported_rx(struct mlx5_core_dev *mdev,
					   u8 proto_type)
{
	switch (proto_type) {
	case IPPROTO_GRE:
		return MLX5_CAP_ETH(mdev, tunnel_stateless_gre);
	case IPPROTO_IPIP:
	case IPPROTO_IPV6:
		return (MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip) ||
			MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip_rx));
	default:
		return false;
	}
}

static bool mlx5_tunnel_any_rx_proto_supported(struct mlx5_core_dev *mdev)
{
	int tt;

	for (tt = 0; tt < MLX5_NUM_TUNNEL_TT; tt++) {
		if (mlx5_tunnel_proto_supported_rx(mdev,
						   ttc_tunnel_rules[tt].proto))
			return true;
	}
	return false;
}

bool mlx5_tunnel_inner_ft_supported(struct mlx5_core_dev *mdev)
{
	return (mlx5_tunnel_any_rx_proto_supported(mdev) &&
		MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
					  ft_field_support.inner_ip_version));
}

static u8 mlx5_etype_to_ipv(u16 ethertype)
{
	if (ethertype == ETH_P_IP)
		return 4;

	if (ethertype == ETH_P_IPV6)
		return 6;

	return 0;
}

static struct mlx5_flow_handle *
mlx5_generate_ttc_rule(struct mlx5_core_dev *dev, struct mlx5_flow_table *ft,
		       struct mlx5_flow_destination *dest, u16 etype, u8 proto)
{
	int match_ipv_outer =
		MLX5_CAP_FLOWTABLE_NIC_RX(dev,
					  ft_field_support.outer_ip_version);
	MLX5_DECLARE_FLOW_ACT(flow_act);
	struct mlx5_flow_handle *rule;
	struct mlx5_flow_spec *spec;
	int err = 0;
	u8 ipv;

	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
	if (!spec)
		return ERR_PTR(-ENOMEM);

	if (proto) {
		spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
		MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, proto);
	}

	ipv = mlx5_etype_to_ipv(etype);
	if (match_ipv_outer && ipv) {
		spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
		MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, ipv);
	} else if (etype) {
		spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype);
		MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, etype);
	}

	rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, 1);
	if (IS_ERR(rule)) {
		err = PTR_ERR(rule);
		mlx5_core_err(dev, "%s: add rule failed\n", __func__);
	}

	kvfree(spec);
	return err ? ERR_PTR(err) : rule;
}

static int mlx5_generate_ttc_table_rules(struct mlx5_core_dev *dev,
					 struct ttc_params *params,
					 struct mlx5_ttc_table *ttc)
{
	struct mlx5_flow_handle **trules;
	struct mlx5_ttc_rule *rules;
	struct mlx5_flow_table *ft;
	int tt;
	int err;

	ft = ttc->t;
	rules = ttc->rules;
	for (tt = 0; tt < MLX5_NUM_TT; tt++) {
		struct mlx5_ttc_rule *rule = &rules[tt];

		rule->rule = mlx5_generate_ttc_rule(dev, ft, &params->dests[tt],
						    ttc_rules[tt].etype,
						    ttc_rules[tt].proto);
		if (IS_ERR(rule->rule)) {
			err = PTR_ERR(rule->rule);
			rule->rule = NULL;
			goto del_rules;
		}
		rule->default_dest = params->dests[tt];
	}

	if (!params->inner_ttc || !mlx5_tunnel_inner_ft_supported(dev))
		return 0;

	trules    = ttc->tunnel_rules;
	for (tt = 0; tt < MLX5_NUM_TUNNEL_TT; tt++) {
		if (!mlx5_tunnel_proto_supported_rx(dev,
						    ttc_tunnel_rules[tt].proto))
			continue;
		trules[tt] = mlx5_generate_ttc_rule(dev, ft,
						    &params->tunnel_dests[tt],
						    ttc_tunnel_rules[tt].etype,
						    ttc_tunnel_rules[tt].proto);
		if (IS_ERR(trules[tt])) {
			err = PTR_ERR(trules[tt]);
			trules[tt] = NULL;
			goto del_rules;
		}
	}

	return 0;

del_rules:
	mlx5_cleanup_ttc_rules(ttc);
	return err;
}

static int mlx5_create_ttc_table_groups(struct mlx5_ttc_table *ttc,
					bool use_ipv)
{
	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
	int ix = 0;
	u32 *in;
	int err;
	u8 *mc;

	ttc->g = kcalloc(MLX5_TTC_NUM_GROUPS, sizeof(*ttc->g), GFP_KERNEL);
	if (!ttc->g)
		return -ENOMEM;
	in = kvzalloc(inlen, GFP_KERNEL);
	if (!in) {
		kfree(ttc->g);
		ttc->g = NULL;
		return -ENOMEM;
	}

	/* L4 Group */
	mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
	MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol);
	if (use_ipv)
		MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_version);
	else
		MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
	MLX5_SET_CFG(in, start_flow_index, ix);
	ix += MLX5_TTC_GROUP1_SIZE;
	MLX5_SET_CFG(in, end_flow_index, ix - 1);
	ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
	if (IS_ERR(ttc->g[ttc->num_groups]))
		goto err;
	ttc->num_groups++;

	/* L3 Group */
	MLX5_SET(fte_match_param, mc, outer_headers.ip_protocol, 0);
	MLX5_SET_CFG(in, start_flow_index, ix);
	ix += MLX5_TTC_GROUP2_SIZE;
	MLX5_SET_CFG(in, end_flow_index, ix - 1);
	ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
	if (IS_ERR(ttc->g[ttc->num_groups]))
		goto err;
	ttc->num_groups++;

	/* Any Group */
	memset(in, 0, inlen);
	MLX5_SET_CFG(in, start_flow_index, ix);
	ix += MLX5_TTC_GROUP3_SIZE;
	MLX5_SET_CFG(in, end_flow_index, ix - 1);
	ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
	if (IS_ERR(ttc->g[ttc->num_groups]))
		goto err;
	ttc->num_groups++;

	kvfree(in);
	return 0;

err:
	err = PTR_ERR(ttc->g[ttc->num_groups]);
	ttc->g[ttc->num_groups] = NULL;
	kvfree(in);

	return err;
}

static struct mlx5_flow_handle *
mlx5_generate_inner_ttc_rule(struct mlx5_core_dev *dev,
			     struct mlx5_flow_table *ft,
			     struct mlx5_flow_destination *dest,
			     u16 etype, u8 proto)
{
	MLX5_DECLARE_FLOW_ACT(flow_act);
	struct mlx5_flow_handle *rule;
	struct mlx5_flow_spec *spec;
	int err = 0;
	u8 ipv;

	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
	if (!spec)
		return ERR_PTR(-ENOMEM);

	ipv = mlx5_etype_to_ipv(etype);
	if (etype && ipv) {
		spec->match_criteria_enable = MLX5_MATCH_INNER_HEADERS;
		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, inner_headers.ip_version);
		MLX5_SET(fte_match_param, spec->match_value, inner_headers.ip_version, ipv);
	}

	if (proto) {
		spec->match_criteria_enable = MLX5_MATCH_INNER_HEADERS;
		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, inner_headers.ip_protocol);
		MLX5_SET(fte_match_param, spec->match_value, inner_headers.ip_protocol, proto);
	}

	rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, 1);
	if (IS_ERR(rule)) {
		err = PTR_ERR(rule);
		mlx5_core_err(dev, "%s: add inner TTC rule failed\n", __func__);
	}

	kvfree(spec);
	return err ? ERR_PTR(err) : rule;
}

static int mlx5_generate_inner_ttc_table_rules(struct mlx5_core_dev *dev,
					       struct ttc_params *params,
					       struct mlx5_ttc_table *ttc)
{
	struct mlx5_ttc_rule *rules;
	struct mlx5_flow_table *ft;
	int err;
	int tt;

	ft = ttc->t;
	rules = ttc->rules;

	for (tt = 0; tt < MLX5_NUM_TT; tt++) {
		struct mlx5_ttc_rule *rule = &rules[tt];

		rule->rule = mlx5_generate_inner_ttc_rule(dev, ft,
							  &params->dests[tt],
							  ttc_rules[tt].etype,
							  ttc_rules[tt].proto);
		if (IS_ERR(rule->rule)) {
			err = PTR_ERR(rule->rule);
			rule->rule = NULL;
			goto del_rules;
		}
		rule->default_dest = params->dests[tt];
	}

	return 0;

del_rules:

	mlx5_cleanup_ttc_rules(ttc);
	return err;
}

static int mlx5_create_inner_ttc_table_groups(struct mlx5_ttc_table *ttc)
{
	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
	int ix = 0;
	u32 *in;
	int err;
	u8 *mc;

	ttc->g = kcalloc(MLX5_INNER_TTC_NUM_GROUPS, sizeof(*ttc->g),
			 GFP_KERNEL);
	if (!ttc->g)
		return -ENOMEM;
	in = kvzalloc(inlen, GFP_KERNEL);
	if (!in) {
		kfree(ttc->g);
		ttc->g = NULL;
		return -ENOMEM;
	}

	/* L4 Group */
	mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
	MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ip_protocol);
	MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ip_version);
	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_INNER_HEADERS);
	MLX5_SET_CFG(in, start_flow_index, ix);
	ix += MLX5_INNER_TTC_GROUP1_SIZE;
	MLX5_SET_CFG(in, end_flow_index, ix - 1);
	ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
	if (IS_ERR(ttc->g[ttc->num_groups]))
		goto err;
	ttc->num_groups++;

	/* L3 Group */
	MLX5_SET(fte_match_param, mc, inner_headers.ip_protocol, 0);
	MLX5_SET_CFG(in, start_flow_index, ix);
	ix += MLX5_INNER_TTC_GROUP2_SIZE;
	MLX5_SET_CFG(in, end_flow_index, ix - 1);
	ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
	if (IS_ERR(ttc->g[ttc->num_groups]))
		goto err;
	ttc->num_groups++;

	/* Any Group */
	memset(in, 0, inlen);
	MLX5_SET_CFG(in, start_flow_index, ix);
	ix += MLX5_INNER_TTC_GROUP3_SIZE;
	MLX5_SET_CFG(in, end_flow_index, ix - 1);
	ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
	if (IS_ERR(ttc->g[ttc->num_groups]))
		goto err;
	ttc->num_groups++;

	kvfree(in);
	return 0;

err:
	err = PTR_ERR(ttc->g[ttc->num_groups]);
	ttc->g[ttc->num_groups] = NULL;
	kvfree(in);

	return err;
}

struct mlx5_ttc_table *mlx5_create_inner_ttc_table(struct mlx5_core_dev *dev,
						   struct ttc_params *params)
{
	struct mlx5_ttc_table *ttc;
	int err;

	ttc = kvzalloc(sizeof(*ttc), GFP_KERNEL);
	if (!ttc)
		return ERR_PTR(-ENOMEM);

	WARN_ON_ONCE(params->ft_attr.max_fte);
	params->ft_attr.max_fte = MLX5_INNER_TTC_TABLE_SIZE;
	ttc->t = mlx5_create_flow_table(params->ns, &params->ft_attr);
	if (IS_ERR(ttc->t)) {
		err = PTR_ERR(ttc->t);
		kvfree(ttc);
		return ERR_PTR(err);
	}

	err = mlx5_create_inner_ttc_table_groups(ttc);
	if (err)
		goto destroy_ft;

	err = mlx5_generate_inner_ttc_table_rules(dev, params, ttc);
	if (err)
		goto destroy_ft;

	return ttc;

destroy_ft:
	mlx5_destroy_ttc_table(ttc);
	return ERR_PTR(err);
}

void mlx5_destroy_ttc_table(struct mlx5_ttc_table *ttc)
{
	int i;

	mlx5_cleanup_ttc_rules(ttc);
	for (i = ttc->num_groups - 1; i >= 0; i--) {
		if (!IS_ERR_OR_NULL(ttc->g[i]))
			mlx5_destroy_flow_group(ttc->g[i]);
		ttc->g[i] = NULL;
	}

	kfree(ttc->g);
	mlx5_destroy_flow_table(ttc->t);
	kvfree(ttc);
}

struct mlx5_ttc_table *mlx5_create_ttc_table(struct mlx5_core_dev *dev,
					     struct ttc_params *params)
{
	bool match_ipv_outer =
		MLX5_CAP_FLOWTABLE_NIC_RX(dev,
					  ft_field_support.outer_ip_version);
	struct mlx5_ttc_table *ttc;
	int err;

	ttc = kvzalloc(sizeof(*ttc), GFP_KERNEL);
	if (!ttc)
		return ERR_PTR(-ENOMEM);

	WARN_ON_ONCE(params->ft_attr.max_fte);
	params->ft_attr.max_fte = MLX5_TTC_TABLE_SIZE;
	ttc->t = mlx5_create_flow_table(params->ns, &params->ft_attr);
	if (IS_ERR(ttc->t)) {
		err = PTR_ERR(ttc->t);
		kvfree(ttc);
		return ERR_PTR(err);
	}

	err = mlx5_create_ttc_table_groups(ttc, match_ipv_outer);
	if (err)
		goto destroy_ft;

	err = mlx5_generate_ttc_table_rules(dev, params, ttc);
	if (err)
		goto destroy_ft;

	return ttc;

destroy_ft:
	mlx5_destroy_ttc_table(ttc);
	return ERR_PTR(err);
}

int mlx5_ttc_fwd_dest(struct mlx5_ttc_table *ttc, enum mlx5_traffic_types type,
		      struct mlx5_flow_destination *new_dest)
{
	return mlx5_modify_rule_destination(ttc->rules[type].rule, new_dest,
					    NULL);
}

struct mlx5_flow_destination
mlx5_ttc_get_default_dest(struct mlx5_ttc_table *ttc,
			  enum mlx5_traffic_types type)
{
	struct mlx5_flow_destination *dest = &ttc->rules[type].default_dest;

	WARN_ONCE(dest->type != MLX5_FLOW_DESTINATION_TYPE_TIR,
		  "TTC[%d] default dest is not setup yet", type);

	return *dest;
}

int mlx5_ttc_fwd_default_dest(struct mlx5_ttc_table *ttc,
			      enum mlx5_traffic_types type)
{
	struct mlx5_flow_destination dest = mlx5_ttc_get_default_dest(ttc, type);

	return mlx5_ttc_fwd_dest(ttc, type, &dest);
}