| Author | Tokens | Token Proportion | Commits | Commit Proportion |
|---|---|---|---|---|
| Leon Romanovsky | 11860 | 99.10% | 4 | 36.36% |
| Yevgeny Kliteynik | 37 | 0.31% | 1 | 9.09% |
| Mark Bloch | 29 | 0.24% | 2 | 18.18% |
| Maor Gottlieb | 25 | 0.21% | 2 | 18.18% |
| Michael Guralnik | 14 | 0.12% | 1 | 9.09% |
| Daria Velikovsky | 3 | 0.03% | 1 | 9.09% |
| Total | 11968 | 11 |
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved. */ #include <rdma/ib_user_verbs.h> #include <rdma/ib_verbs.h> #include <rdma/uverbs_types.h> #include <rdma/uverbs_ioctl.h> #include <rdma/uverbs_std_types.h> #include <rdma/mlx5_user_ioctl_cmds.h> #include <rdma/mlx5_user_ioctl_verbs.h> #include <rdma/ib_umem.h> #include <linux/mlx5/driver.h> #include <linux/mlx5/fs.h> #include <linux/mlx5/fs_helpers.h> #include <linux/mlx5/accel.h> #include <linux/mlx5/eswitch.h> #include "mlx5_ib.h" #include "counters.h" #include "devx.h" #include "fs.h" #define UVERBS_MODULE_NAME mlx5_ib #include <rdma/uverbs_named_ioctl.h> enum { MATCH_CRITERIA_ENABLE_OUTER_BIT, MATCH_CRITERIA_ENABLE_MISC_BIT, MATCH_CRITERIA_ENABLE_INNER_BIT, MATCH_CRITERIA_ENABLE_MISC2_BIT }; #define HEADER_IS_ZERO(match_criteria, headers) \ !(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \ 0, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \ static u8 get_match_criteria_enable(u32 *match_criteria) { u8 match_criteria_enable; match_criteria_enable = (!HEADER_IS_ZERO(match_criteria, outer_headers)) << MATCH_CRITERIA_ENABLE_OUTER_BIT; match_criteria_enable |= (!HEADER_IS_ZERO(match_criteria, misc_parameters)) << MATCH_CRITERIA_ENABLE_MISC_BIT; match_criteria_enable |= (!HEADER_IS_ZERO(match_criteria, inner_headers)) << MATCH_CRITERIA_ENABLE_INNER_BIT; match_criteria_enable |= (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) << MATCH_CRITERIA_ENABLE_MISC2_BIT; return match_criteria_enable; } static int set_proto(void *outer_c, void *outer_v, u8 mask, u8 val) { u8 entry_mask; u8 entry_val; int err = 0; if (!mask) goto out; entry_mask = MLX5_GET(fte_match_set_lyr_2_4, outer_c, ip_protocol); entry_val = MLX5_GET(fte_match_set_lyr_2_4, outer_v, ip_protocol); if (!entry_mask) { MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask); MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val); goto out; } /* Don't override existing ip protocol */ if (mask != entry_mask || val != entry_val) err = -EINVAL; out: return err; } static void set_flow_label(void *misc_c, void *misc_v, u32 mask, u32 val, bool inner) { if (inner) { MLX5_SET(fte_match_set_misc, misc_c, inner_ipv6_flow_label, mask); MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label, val); } else { MLX5_SET(fte_match_set_misc, misc_c, outer_ipv6_flow_label, mask); MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label, val); } } static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val) { MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask); MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val); MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2); MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2); } static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask) { if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) && !(field_support & MLX5_FIELD_SUPPORT_MPLS_LABEL)) return -EOPNOTSUPP; if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) && !(field_support & MLX5_FIELD_SUPPORT_MPLS_EXP)) return -EOPNOTSUPP; if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) && !(field_support & MLX5_FIELD_SUPPORT_MPLS_S_BOS)) return -EOPNOTSUPP; if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) && !(field_support & MLX5_FIELD_SUPPORT_MPLS_TTL)) return -EOPNOTSUPP; return 0; } #define LAST_ETH_FIELD vlan_tag #define LAST_IB_FIELD sl #define LAST_IPV4_FIELD tos #define LAST_IPV6_FIELD traffic_class #define LAST_TCP_UDP_FIELD src_port #define LAST_TUNNEL_FIELD tunnel_id #define LAST_FLOW_TAG_FIELD tag_id #define LAST_DROP_FIELD size #define LAST_COUNTERS_FIELD counters /* Field is the last supported field */ #define FIELDS_NOT_SUPPORTED(filter, field) \ memchr_inv((void *)&filter.field + sizeof(filter.field), 0, \ sizeof(filter) - offsetofend(typeof(filter), field)) int parse_flow_flow_action(struct mlx5_ib_flow_action *maction, bool is_egress, struct mlx5_flow_act *action) { switch (maction->ib_action.type) { case IB_FLOW_ACTION_ESP: if (action->action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT | MLX5_FLOW_CONTEXT_ACTION_DECRYPT)) return -EINVAL; /* Currently only AES_GCM keymat is supported by the driver */ action->esp_id = (uintptr_t)maction->esp_aes_gcm.ctx; action->action |= is_egress ? MLX5_FLOW_CONTEXT_ACTION_ENCRYPT : MLX5_FLOW_CONTEXT_ACTION_DECRYPT; return 0; case IB_FLOW_ACTION_UNSPECIFIED: if (maction->flow_action_raw.sub_type == MLX5_IB_FLOW_ACTION_MODIFY_HEADER) { if (action->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) return -EINVAL; action->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; action->modify_hdr = maction->flow_action_raw.modify_hdr; return 0; } if (maction->flow_action_raw.sub_type == MLX5_IB_FLOW_ACTION_DECAP) { if (action->action & MLX5_FLOW_CONTEXT_ACTION_DECAP) return -EINVAL; action->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP; return 0; } if (maction->flow_action_raw.sub_type == MLX5_IB_FLOW_ACTION_PACKET_REFORMAT) { if (action->action & MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT) return -EINVAL; action->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; action->pkt_reformat = maction->flow_action_raw.pkt_reformat; return 0; } fallthrough; default: return -EOPNOTSUPP; } } static int parse_flow_attr(struct mlx5_core_dev *mdev, struct mlx5_flow_spec *spec, const union ib_flow_spec *ib_spec, const struct ib_flow_attr *flow_attr, struct mlx5_flow_act *action, u32 prev_type) { struct mlx5_flow_context *flow_context = &spec->flow_context; u32 *match_c = spec->match_criteria; u32 *match_v = spec->match_value; void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c, misc_parameters); void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v, misc_parameters); void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c, misc_parameters_2); void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v, misc_parameters_2); void *headers_c; void *headers_v; int match_ipv; int ret; if (ib_spec->type & IB_FLOW_SPEC_INNER) { headers_c = MLX5_ADDR_OF(fte_match_param, match_c, inner_headers); headers_v = MLX5_ADDR_OF(fte_match_param, match_v, inner_headers); match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ft_field_support.inner_ip_version); } else { headers_c = MLX5_ADDR_OF(fte_match_param, match_c, outer_headers); headers_v = MLX5_ADDR_OF(fte_match_param, match_v, outer_headers); match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ft_field_support.outer_ip_version); } switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) { case IB_FLOW_SPEC_ETH: if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD)) return -EOPNOTSUPP; ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, dmac_47_16), ib_spec->eth.mask.dst_mac); ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16), ib_spec->eth.val.dst_mac); ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, smac_47_16), ib_spec->eth.mask.src_mac); ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16), ib_spec->eth.val.src_mac); if (ib_spec->eth.mask.vlan_tag) { MLX5_SET(fte_match_set_lyr_2_4, headers_c, cvlan_tag, 1); MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1); MLX5_SET(fte_match_set_lyr_2_4, headers_c, first_vid, ntohs(ib_spec->eth.mask.vlan_tag)); MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, ntohs(ib_spec->eth.val.vlan_tag)); MLX5_SET(fte_match_set_lyr_2_4, headers_c, first_cfi, ntohs(ib_spec->eth.mask.vlan_tag) >> 12); MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, ntohs(ib_spec->eth.val.vlan_tag) >> 12); MLX5_SET(fte_match_set_lyr_2_4, headers_c, first_prio, ntohs(ib_spec->eth.mask.vlan_tag) >> 13); MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, ntohs(ib_spec->eth.val.vlan_tag) >> 13); } MLX5_SET(fte_match_set_lyr_2_4, headers_c, ethertype, ntohs(ib_spec->eth.mask.ether_type)); MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, ntohs(ib_spec->eth.val.ether_type)); break; case IB_FLOW_SPEC_IPV4: if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD)) return -EOPNOTSUPP; if (match_ipv) { MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_version, 0xf); MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, MLX5_FS_IPV4_VERSION); } else { MLX5_SET(fte_match_set_lyr_2_4, headers_c, ethertype, 0xffff); MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, ETH_P_IP); } memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, src_ipv4_src_ipv6.ipv4_layout.ipv4), &ib_spec->ipv4.mask.src_ip, sizeof(ib_spec->ipv4.mask.src_ip)); memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, src_ipv4_src_ipv6.ipv4_layout.ipv4), &ib_spec->ipv4.val.src_ip, sizeof(ib_spec->ipv4.val.src_ip)); memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, dst_ipv4_dst_ipv6.ipv4_layout.ipv4), &ib_spec->ipv4.mask.dst_ip, sizeof(ib_spec->ipv4.mask.dst_ip)); memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dst_ipv4_dst_ipv6.ipv4_layout.ipv4), &ib_spec->ipv4.val.dst_ip, sizeof(ib_spec->ipv4.val.dst_ip)); set_tos(headers_c, headers_v, ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos); if (set_proto(headers_c, headers_v, ib_spec->ipv4.mask.proto, ib_spec->ipv4.val.proto)) return -EINVAL; break; case IB_FLOW_SPEC_IPV6: if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD)) return -EOPNOTSUPP; if (match_ipv) { MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_version, 0xf); MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, MLX5_FS_IPV6_VERSION); } else { MLX5_SET(fte_match_set_lyr_2_4, headers_c, ethertype, 0xffff); MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, ETH_P_IPV6); } memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, src_ipv4_src_ipv6.ipv6_layout.ipv6), &ib_spec->ipv6.mask.src_ip, sizeof(ib_spec->ipv6.mask.src_ip)); memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, src_ipv4_src_ipv6.ipv6_layout.ipv6), &ib_spec->ipv6.val.src_ip, sizeof(ib_spec->ipv6.val.src_ip)); memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, dst_ipv4_dst_ipv6.ipv6_layout.ipv6), &ib_spec->ipv6.mask.dst_ip, sizeof(ib_spec->ipv6.mask.dst_ip)); memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dst_ipv4_dst_ipv6.ipv6_layout.ipv6), &ib_spec->ipv6.val.dst_ip, sizeof(ib_spec->ipv6.val.dst_ip)); set_tos(headers_c, headers_v, ib_spec->ipv6.mask.traffic_class, ib_spec->ipv6.val.traffic_class); if (set_proto(headers_c, headers_v, ib_spec->ipv6.mask.next_hdr, ib_spec->ipv6.val.next_hdr)) return -EINVAL; set_flow_label(misc_params_c, misc_params_v, ntohl(ib_spec->ipv6.mask.flow_label), ntohl(ib_spec->ipv6.val.flow_label), ib_spec->type & IB_FLOW_SPEC_INNER); break; case IB_FLOW_SPEC_ESP: if (ib_spec->esp.mask.seq) return -EOPNOTSUPP; MLX5_SET(fte_match_set_misc, misc_params_c, outer_esp_spi, ntohl(ib_spec->esp.mask.spi)); MLX5_SET(fte_match_set_misc, misc_params_v, outer_esp_spi, ntohl(ib_spec->esp.val.spi)); break; case IB_FLOW_SPEC_TCP: if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask, LAST_TCP_UDP_FIELD)) return -EOPNOTSUPP; if (set_proto(headers_c, headers_v, 0xff, IPPROTO_TCP)) return -EINVAL; MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport, ntohs(ib_spec->tcp_udp.mask.src_port)); MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport, ntohs(ib_spec->tcp_udp.val.src_port)); MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport, ntohs(ib_spec->tcp_udp.mask.dst_port)); MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport, ntohs(ib_spec->tcp_udp.val.dst_port)); break; case IB_FLOW_SPEC_UDP: if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask, LAST_TCP_UDP_FIELD)) return -EOPNOTSUPP; if (set_proto(headers_c, headers_v, 0xff, IPPROTO_UDP)) return -EINVAL; MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport, ntohs(ib_spec->tcp_udp.mask.src_port)); MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport, ntohs(ib_spec->tcp_udp.val.src_port)); MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport, ntohs(ib_spec->tcp_udp.mask.dst_port)); MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, ntohs(ib_spec->tcp_udp.val.dst_port)); break; case IB_FLOW_SPEC_GRE: if (ib_spec->gre.mask.c_ks_res0_ver) return -EOPNOTSUPP; if (set_proto(headers_c, headers_v, 0xff, IPPROTO_GRE)) return -EINVAL; MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol, 0xff); MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE); MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol, ntohs(ib_spec->gre.mask.protocol)); MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol, ntohs(ib_spec->gre.val.protocol)); memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c, gre_key.nvgre.hi), &ib_spec->gre.mask.key, sizeof(ib_spec->gre.mask.key)); memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v, gre_key.nvgre.hi), &ib_spec->gre.val.key, sizeof(ib_spec->gre.val.key)); break; case IB_FLOW_SPEC_MPLS: switch (prev_type) { case IB_FLOW_SPEC_UDP: if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ft_field_support.outer_first_mpls_over_udp), &ib_spec->mpls.mask.tag)) return -EOPNOTSUPP; memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, outer_first_mpls_over_udp), &ib_spec->mpls.val.tag, sizeof(ib_spec->mpls.val.tag)); memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, outer_first_mpls_over_udp), &ib_spec->mpls.mask.tag, sizeof(ib_spec->mpls.mask.tag)); break; case IB_FLOW_SPEC_GRE: if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ft_field_support.outer_first_mpls_over_gre), &ib_spec->mpls.mask.tag)) return -EOPNOTSUPP; memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, outer_first_mpls_over_gre), &ib_spec->mpls.val.tag, sizeof(ib_spec->mpls.val.tag)); memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, outer_first_mpls_over_gre), &ib_spec->mpls.mask.tag, sizeof(ib_spec->mpls.mask.tag)); break; default: if (ib_spec->type & IB_FLOW_SPEC_INNER) { if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ft_field_support.inner_first_mpls), &ib_spec->mpls.mask.tag)) return -EOPNOTSUPP; memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, inner_first_mpls), &ib_spec->mpls.val.tag, sizeof(ib_spec->mpls.val.tag)); memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, inner_first_mpls), &ib_spec->mpls.mask.tag, sizeof(ib_spec->mpls.mask.tag)); } else { if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ft_field_support.outer_first_mpls), &ib_spec->mpls.mask.tag)) return -EOPNOTSUPP; memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, outer_first_mpls), &ib_spec->mpls.val.tag, sizeof(ib_spec->mpls.val.tag)); memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, outer_first_mpls), &ib_spec->mpls.mask.tag, sizeof(ib_spec->mpls.mask.tag)); } } break; case IB_FLOW_SPEC_VXLAN_TUNNEL: if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask, LAST_TUNNEL_FIELD)) return -EOPNOTSUPP; MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni, ntohl(ib_spec->tunnel.mask.tunnel_id)); MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni, ntohl(ib_spec->tunnel.val.tunnel_id)); break; case IB_FLOW_SPEC_ACTION_TAG: if (FIELDS_NOT_SUPPORTED(ib_spec->flow_tag, LAST_FLOW_TAG_FIELD)) return -EOPNOTSUPP; if (ib_spec->flow_tag.tag_id >= BIT(24)) return -EINVAL; flow_context->flow_tag = ib_spec->flow_tag.tag_id; flow_context->flags |= FLOW_CONTEXT_HAS_TAG; break; case IB_FLOW_SPEC_ACTION_DROP: if (FIELDS_NOT_SUPPORTED(ib_spec->drop, LAST_DROP_FIELD)) return -EOPNOTSUPP; action->action |= MLX5_FLOW_CONTEXT_ACTION_DROP; break; case IB_FLOW_SPEC_ACTION_HANDLE: ret = parse_flow_flow_action(to_mflow_act(ib_spec->action.act), flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS, action); if (ret) return ret; break; case IB_FLOW_SPEC_ACTION_COUNT: if (FIELDS_NOT_SUPPORTED(ib_spec->flow_count, LAST_COUNTERS_FIELD)) return -EOPNOTSUPP; /* for now support only one counters spec per flow */ if (action->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) return -EINVAL; action->counters = ib_spec->flow_count.counters; action->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; break; default: return -EINVAL; } return 0; } /* If a flow could catch both multicast and unicast packets, * it won't fall into the multicast flow steering table and this rule * could steal other multicast packets. */ static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr) { union ib_flow_spec *flow_spec; if (ib_attr->type != IB_FLOW_ATTR_NORMAL || ib_attr->num_of_specs < 1) return false; flow_spec = (union ib_flow_spec *)(ib_attr + 1); if (flow_spec->type == IB_FLOW_SPEC_IPV4) { struct ib_flow_spec_ipv4 *ipv4_spec; ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec; if (ipv4_is_multicast(ipv4_spec->val.dst_ip)) return true; return false; } if (flow_spec->type == IB_FLOW_SPEC_ETH) { struct ib_flow_spec_eth *eth_spec; eth_spec = (struct ib_flow_spec_eth *)flow_spec; return is_multicast_ether_addr(eth_spec->mask.dst_mac) && is_multicast_ether_addr(eth_spec->val.dst_mac); } return false; } enum valid_spec { VALID_SPEC_INVALID, VALID_SPEC_VALID, VALID_SPEC_NA, }; static enum valid_spec is_valid_esp_aes_gcm(struct mlx5_core_dev *mdev, const struct mlx5_flow_spec *spec, const struct mlx5_flow_act *flow_act, bool egress) { const u32 *match_c = spec->match_criteria; bool is_crypto = (flow_act->action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT | MLX5_FLOW_CONTEXT_ACTION_DECRYPT)); bool is_ipsec = mlx5_fs_is_ipsec_flow(match_c); bool is_drop = flow_act->action & MLX5_FLOW_CONTEXT_ACTION_DROP; /* * Currently only crypto is supported in egress, when regular egress * rules would be supported, always return VALID_SPEC_NA. */ if (!is_crypto) return VALID_SPEC_NA; return is_crypto && is_ipsec && (!egress || (!is_drop && !(spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG))) ? VALID_SPEC_VALID : VALID_SPEC_INVALID; } static bool is_valid_spec(struct mlx5_core_dev *mdev, const struct mlx5_flow_spec *spec, const struct mlx5_flow_act *flow_act, bool egress) { /* We curretly only support ipsec egress flow */ return is_valid_esp_aes_gcm(mdev, spec, flow_act, egress) != VALID_SPEC_INVALID; } static bool is_valid_ethertype(struct mlx5_core_dev *mdev, const struct ib_flow_attr *flow_attr, bool check_inner) { union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1); int match_ipv = check_inner ? MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ft_field_support.inner_ip_version) : MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ft_field_support.outer_ip_version); int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0; bool ipv4_spec_valid, ipv6_spec_valid; unsigned int ip_spec_type = 0; bool has_ethertype = false; unsigned int spec_index; bool mask_valid = true; u16 eth_type = 0; bool type_valid; /* Validate that ethertype is correct */ for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) { if ((ib_spec->type == (IB_FLOW_SPEC_ETH | inner_bit)) && ib_spec->eth.mask.ether_type) { mask_valid = (ib_spec->eth.mask.ether_type == htons(0xffff)); has_ethertype = true; eth_type = ntohs(ib_spec->eth.val.ether_type); } else if ((ib_spec->type == (IB_FLOW_SPEC_IPV4 | inner_bit)) || (ib_spec->type == (IB_FLOW_SPEC_IPV6 | inner_bit))) { ip_spec_type = ib_spec->type; } ib_spec = (void *)ib_spec + ib_spec->size; } type_valid = (!has_ethertype) || (!ip_spec_type); if (!type_valid && mask_valid) { ipv4_spec_valid = (eth_type == ETH_P_IP) && (ip_spec_type == (IB_FLOW_SPEC_IPV4 | inner_bit)); ipv6_spec_valid = (eth_type == ETH_P_IPV6) && (ip_spec_type == (IB_FLOW_SPEC_IPV6 | inner_bit)); type_valid = (ipv4_spec_valid) || (ipv6_spec_valid) || (((eth_type == ETH_P_MPLS_UC) || (eth_type == ETH_P_MPLS_MC)) && match_ipv); } return type_valid; } static bool is_valid_attr(struct mlx5_core_dev *mdev, const struct ib_flow_attr *flow_attr) { return is_valid_ethertype(mdev, flow_attr, false) && is_valid_ethertype(mdev, flow_attr, true); } static void put_flow_table(struct mlx5_ib_dev *dev, struct mlx5_ib_flow_prio *prio, bool ft_added) { prio->refcount -= !!ft_added; if (!prio->refcount) { mlx5_destroy_flow_table(prio->flow_table); prio->flow_table = NULL; } } static int mlx5_ib_destroy_flow(struct ib_flow *flow_id) { struct mlx5_ib_flow_handler *handler = container_of(flow_id, struct mlx5_ib_flow_handler, ibflow); struct mlx5_ib_flow_handler *iter, *tmp; struct mlx5_ib_dev *dev = handler->dev; mutex_lock(&dev->flow_db->lock); list_for_each_entry_safe(iter, tmp, &handler->list, list) { mlx5_del_flow_rules(iter->rule); put_flow_table(dev, iter->prio, true); list_del(&iter->list); kfree(iter); } mlx5_del_flow_rules(handler->rule); put_flow_table(dev, handler->prio, true); mlx5_ib_counters_clear_description(handler->ibcounters); mutex_unlock(&dev->flow_db->lock); if (handler->flow_matcher) atomic_dec(&handler->flow_matcher->usecnt); kfree(handler); return 0; } static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap) { priority *= 2; if (!dont_trap) priority++; return priority; } enum flow_table_type { MLX5_IB_FT_RX, MLX5_IB_FT_TX }; #define MLX5_FS_MAX_TYPES 6 #define MLX5_FS_MAX_ENTRIES BIT(16) static struct mlx5_ib_flow_prio *_get_prio(struct mlx5_flow_namespace *ns, struct mlx5_ib_flow_prio *prio, int priority, int num_entries, int num_groups, u32 flags) { struct mlx5_flow_table_attr ft_attr = {}; struct mlx5_flow_table *ft; ft_attr.prio = priority; ft_attr.max_fte = num_entries; ft_attr.flags = flags; ft_attr.autogroup.max_num_groups = num_groups; ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); if (IS_ERR(ft)) return ERR_CAST(ft); prio->flow_table = ft; prio->refcount = 0; return prio; } static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, struct ib_flow_attr *flow_attr, enum flow_table_type ft_type) { bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP; struct mlx5_flow_namespace *ns = NULL; enum mlx5_flow_namespace_type fn_type; struct mlx5_ib_flow_prio *prio; struct mlx5_flow_table *ft; int max_table_size; int num_entries; int num_groups; bool esw_encap; u32 flags = 0; int priority; max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, log_max_ft_size)); esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != DEVLINK_ESWITCH_ENCAP_MODE_NONE; switch (flow_attr->type) { case IB_FLOW_ATTR_NORMAL: if (flow_is_multicast_only(flow_attr) && !dont_trap) priority = MLX5_IB_FLOW_MCAST_PRIO; else priority = ib_prio_to_core_prio(flow_attr->priority, dont_trap); if (ft_type == MLX5_IB_FT_RX) { fn_type = MLX5_FLOW_NAMESPACE_BYPASS; prio = &dev->flow_db->prios[priority]; if (!dev->is_rep && !esw_encap && MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap)) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; if (!dev->is_rep && !esw_encap && MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, reformat_l3_tunnel_to_l2)) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; } else { max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_TX( dev->mdev, log_max_ft_size)); fn_type = MLX5_FLOW_NAMESPACE_EGRESS; prio = &dev->flow_db->egress_prios[priority]; if (!dev->is_rep && !esw_encap && MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat)) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; } ns = mlx5_get_flow_namespace(dev->mdev, fn_type); num_entries = MLX5_FS_MAX_ENTRIES; num_groups = MLX5_FS_MAX_TYPES; break; case IB_FLOW_ATTR_ALL_DEFAULT: case IB_FLOW_ATTR_MC_DEFAULT: ns = mlx5_get_flow_namespace(dev->mdev, MLX5_FLOW_NAMESPACE_LEFTOVERS); build_leftovers_ft_param(&priority, &num_entries, &num_groups); prio = &dev->flow_db->prios[MLX5_IB_FLOW_LEFTOVERS_PRIO]; break; case IB_FLOW_ATTR_SNIFFER: if (!MLX5_CAP_FLOWTABLE(dev->mdev, allow_sniffer_and_nic_rx_shared_tir)) return ERR_PTR(-EOPNOTSUPP); ns = mlx5_get_flow_namespace( dev->mdev, ft_type == MLX5_IB_FT_RX ? MLX5_FLOW_NAMESPACE_SNIFFER_RX : MLX5_FLOW_NAMESPACE_SNIFFER_TX); prio = &dev->flow_db->sniffer[ft_type]; priority = 0; num_entries = 1; num_groups = 1; break; default: break; } if (!ns) return ERR_PTR(-EOPNOTSUPP); max_table_size = min_t(int, num_entries, max_table_size); ft = prio->flow_table; if (!ft) return _get_prio(ns, prio, priority, max_table_size, num_groups, flags); return prio; } static void set_underlay_qp(struct mlx5_ib_dev *dev, struct mlx5_flow_spec *spec, u32 underlay_qpn) { void *misc_params_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); if (underlay_qpn && MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, ft_field_support.bth_dst_qp)) { MLX5_SET(fte_match_set_misc, misc_params_v, bth_dst_qp, underlay_qpn); MLX5_SET(fte_match_set_misc, misc_params_c, bth_dst_qp, 0xffffff); } } static void mlx5_ib_set_rule_source_port(struct mlx5_ib_dev *dev, struct mlx5_flow_spec *spec, struct mlx5_eswitch_rep *rep) { struct mlx5_eswitch *esw = dev->mdev->priv.eswitch; void *misc; if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_for_match(rep->esw, rep->vport)); misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask()); } else { misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); MLX5_SET(fte_match_set_misc, misc, source_port, rep->vport); misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); } } static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev, struct mlx5_ib_flow_prio *ft_prio, const struct ib_flow_attr *flow_attr, struct mlx5_flow_destination *dst, u32 underlay_qpn, struct mlx5_ib_create_flow *ucmd) { struct mlx5_flow_table *ft = ft_prio->flow_table; struct mlx5_ib_flow_handler *handler; struct mlx5_flow_act flow_act = {}; struct mlx5_flow_spec *spec; struct mlx5_flow_destination dest_arr[2] = {}; struct mlx5_flow_destination *rule_dst = dest_arr; const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr); unsigned int spec_index; u32 prev_type = 0; int err = 0; int dest_num = 0; bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS; if (!is_valid_attr(dev->mdev, flow_attr)) return ERR_PTR(-EINVAL); if (dev->is_rep && is_egress) return ERR_PTR(-EINVAL); spec = kvzalloc(sizeof(*spec), GFP_KERNEL); handler = kzalloc(sizeof(*handler), GFP_KERNEL); if (!handler || !spec) { err = -ENOMEM; goto free; } INIT_LIST_HEAD(&handler->list); for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) { err = parse_flow_attr(dev->mdev, spec, ib_flow, flow_attr, &flow_act, prev_type); if (err < 0) goto free; prev_type = ((union ib_flow_spec *)ib_flow)->type; ib_flow += ((union ib_flow_spec *)ib_flow)->size; } if (dst && !(flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP)) { memcpy(&dest_arr[0], dst, sizeof(*dst)); dest_num++; } if (!flow_is_multicast_only(flow_attr)) set_underlay_qp(dev, spec, underlay_qpn); if (dev->is_rep && flow_attr->type != IB_FLOW_ATTR_SNIFFER) { struct mlx5_eswitch_rep *rep; rep = dev->port[flow_attr->port - 1].rep; if (!rep) { err = -EINVAL; goto free; } mlx5_ib_set_rule_source_port(dev, spec, rep); } spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria); if (is_egress && !is_valid_spec(dev->mdev, spec, &flow_act, is_egress)) { err = -EINVAL; goto free; } if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { struct mlx5_ib_mcounters *mcounters; err = mlx5_ib_flow_counters_set_data(flow_act.counters, ucmd); if (err) goto free; mcounters = to_mcounters(flow_act.counters); handler->ibcounters = flow_act.counters; dest_arr[dest_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; dest_arr[dest_num].counter_id = mlx5_fc_id(mcounters->hw_cntrs_hndl); dest_num++; } if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP) { if (!dest_num) rule_dst = NULL; } else { if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO; if (is_egress) flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW; else if (dest_num) flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; } if ((spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG) && (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) { mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n", spec->flow_context.flow_tag, flow_attr->type); err = -EINVAL; goto free; } handler->rule = mlx5_add_flow_rules(ft, spec, &flow_act, rule_dst, dest_num); if (IS_ERR(handler->rule)) { err = PTR_ERR(handler->rule); goto free; } ft_prio->refcount++; handler->prio = ft_prio; handler->dev = dev; ft_prio->flow_table = ft; free: if (err && handler) { mlx5_ib_counters_clear_description(handler->ibcounters); kfree(handler); } kvfree(spec); return err ? ERR_PTR(err) : handler; } static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev, struct mlx5_ib_flow_prio *ft_prio, const struct ib_flow_attr *flow_attr, struct mlx5_flow_destination *dst) { return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL); } enum { LEFTOVERS_MC, LEFTOVERS_UC, }; static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev, struct mlx5_ib_flow_prio *ft_prio, struct ib_flow_attr *flow_attr, struct mlx5_flow_destination *dst) { struct mlx5_ib_flow_handler *handler_ucast = NULL; struct mlx5_ib_flow_handler *handler = NULL; static struct { struct ib_flow_attr flow_attr; struct ib_flow_spec_eth eth_flow; } leftovers_specs[] = { [LEFTOVERS_MC] = { .flow_attr = { .num_of_specs = 1, .size = sizeof(leftovers_specs[0]) }, .eth_flow = { .type = IB_FLOW_SPEC_ETH, .size = sizeof(struct ib_flow_spec_eth), .mask = {.dst_mac = {0x1} }, .val = {.dst_mac = {0x1} } } }, [LEFTOVERS_UC] = { .flow_attr = { .num_of_specs = 1, .size = sizeof(leftovers_specs[0]) }, .eth_flow = { .type = IB_FLOW_SPEC_ETH, .size = sizeof(struct ib_flow_spec_eth), .mask = {.dst_mac = {0x1} }, .val = {.dst_mac = {} } } } }; handler = create_flow_rule(dev, ft_prio, &leftovers_specs[LEFTOVERS_MC].flow_attr, dst); if (!IS_ERR(handler) && flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) { handler_ucast = create_flow_rule(dev, ft_prio, &leftovers_specs[LEFTOVERS_UC].flow_attr, dst); if (IS_ERR(handler_ucast)) { mlx5_del_flow_rules(handler->rule); ft_prio->refcount--; kfree(handler); handler = handler_ucast; } else { list_add(&handler_ucast->list, &handler->list); } } return handler; } static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev, struct mlx5_ib_flow_prio *ft_rx, struct mlx5_ib_flow_prio *ft_tx, struct mlx5_flow_destination *dst) { struct mlx5_ib_flow_handler *handler_rx; struct mlx5_ib_flow_handler *handler_tx; int err; static const struct ib_flow_attr flow_attr = { .num_of_specs = 0, .type = IB_FLOW_ATTR_SNIFFER, .size = sizeof(flow_attr) }; handler_rx = create_flow_rule(dev, ft_rx, &flow_attr, dst); if (IS_ERR(handler_rx)) { err = PTR_ERR(handler_rx); goto err; } handler_tx = create_flow_rule(dev, ft_tx, &flow_attr, dst); if (IS_ERR(handler_tx)) { err = PTR_ERR(handler_tx); goto err_tx; } list_add(&handler_tx->list, &handler_rx->list); return handler_rx; err_tx: mlx5_del_flow_rules(handler_rx->rule); ft_rx->refcount--; kfree(handler_rx); err: return ERR_PTR(err); } static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp, struct ib_flow_attr *flow_attr, struct ib_udata *udata) { struct mlx5_ib_dev *dev = to_mdev(qp->device); struct mlx5_ib_qp *mqp = to_mqp(qp); struct mlx5_ib_flow_handler *handler = NULL; struct mlx5_flow_destination *dst = NULL; struct mlx5_ib_flow_prio *ft_prio_tx = NULL; struct mlx5_ib_flow_prio *ft_prio; bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS; struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr; size_t min_ucmd_sz, required_ucmd_sz; int err; int underlay_qpn; if (udata && udata->inlen) { min_ucmd_sz = offsetofend(struct mlx5_ib_create_flow, reserved); if (udata->inlen < min_ucmd_sz) return ERR_PTR(-EOPNOTSUPP); err = ib_copy_from_udata(&ucmd_hdr, udata, min_ucmd_sz); if (err) return ERR_PTR(err); /* currently supports only one counters data */ if (ucmd_hdr.ncounters_data > 1) return ERR_PTR(-EINVAL); required_ucmd_sz = min_ucmd_sz + sizeof(struct mlx5_ib_flow_counters_data) * ucmd_hdr.ncounters_data; if (udata->inlen > required_ucmd_sz && !ib_is_udata_cleared(udata, required_ucmd_sz, udata->inlen - required_ucmd_sz)) return ERR_PTR(-EOPNOTSUPP); ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL); if (!ucmd) return ERR_PTR(-ENOMEM); err = ib_copy_from_udata(ucmd, udata, required_ucmd_sz); if (err) goto free_ucmd; } if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO) { err = -ENOMEM; goto free_ucmd; } if (flow_attr->flags & ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP | IB_FLOW_ATTR_FLAGS_EGRESS)) { err = -EINVAL; goto free_ucmd; } if (is_egress && (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) { err = -EINVAL; goto free_ucmd; } dst = kzalloc(sizeof(*dst), GFP_KERNEL); if (!dst) { err = -ENOMEM; goto free_ucmd; } mutex_lock(&dev->flow_db->lock); ft_prio = get_flow_table(dev, flow_attr, is_egress ? MLX5_IB_FT_TX : MLX5_IB_FT_RX); if (IS_ERR(ft_prio)) { err = PTR_ERR(ft_prio); goto unlock; } if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) { ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX); if (IS_ERR(ft_prio_tx)) { err = PTR_ERR(ft_prio_tx); ft_prio_tx = NULL; goto destroy_ft; } } if (is_egress) { dst->type = MLX5_FLOW_DESTINATION_TYPE_PORT; } else { dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR; if (mqp->is_rss) dst->tir_num = mqp->rss_qp.tirn; else dst->tir_num = mqp->raw_packet_qp.rq.tirn; } switch (flow_attr->type) { case IB_FLOW_ATTR_NORMAL: underlay_qpn = (mqp->flags & IB_QP_CREATE_SOURCE_QPN) ? mqp->underlay_qpn : 0; handler = _create_flow_rule(dev, ft_prio, flow_attr, dst, underlay_qpn, ucmd); break; case IB_FLOW_ATTR_ALL_DEFAULT: case IB_FLOW_ATTR_MC_DEFAULT: handler = create_leftovers_rule(dev, ft_prio, flow_attr, dst); break; case IB_FLOW_ATTR_SNIFFER: handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst); break; default: err = -EINVAL; goto destroy_ft; } if (IS_ERR(handler)) { err = PTR_ERR(handler); handler = NULL; goto destroy_ft; } mutex_unlock(&dev->flow_db->lock); kfree(dst); kfree(ucmd); return &handler->ibflow; destroy_ft: put_flow_table(dev, ft_prio, false); if (ft_prio_tx) put_flow_table(dev, ft_prio_tx, false); unlock: mutex_unlock(&dev->flow_db->lock); kfree(dst); free_ucmd: kfree(ucmd); return ERR_PTR(err); } static struct mlx5_ib_flow_prio * _get_flow_table(struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher, bool mcast) { struct mlx5_flow_namespace *ns = NULL; struct mlx5_ib_flow_prio *prio = NULL; int max_table_size = 0; bool esw_encap; u32 flags = 0; int priority; if (mcast) priority = MLX5_IB_FLOW_MCAST_PRIO; else priority = ib_prio_to_core_prio(fs_matcher->priority, false); esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != DEVLINK_ESWITCH_ENCAP_MODE_NONE; switch (fs_matcher->ns_type) { case MLX5_FLOW_NAMESPACE_BYPASS: max_table_size = BIT( MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, log_max_ft_size)); if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, reformat_l3_tunnel_to_l2) && !esw_encap) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; break; case MLX5_FLOW_NAMESPACE_EGRESS: max_table_size = BIT( MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size)); if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) && !esw_encap) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; break; case MLX5_FLOW_NAMESPACE_FDB: max_table_size = BIT( MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size)); if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, reformat_l3_tunnel_to_l2) && esw_encap) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; priority = FDB_BYPASS_PATH; break; case MLX5_FLOW_NAMESPACE_RDMA_RX: max_table_size = BIT( MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, log_max_ft_size)); priority = fs_matcher->priority; break; case MLX5_FLOW_NAMESPACE_RDMA_TX: max_table_size = BIT( MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, log_max_ft_size)); priority = fs_matcher->priority; break; default: break; } max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES); ns = mlx5_get_flow_namespace(dev->mdev, fs_matcher->ns_type); if (!ns) return ERR_PTR(-EOPNOTSUPP); switch (fs_matcher->ns_type) { case MLX5_FLOW_NAMESPACE_BYPASS: prio = &dev->flow_db->prios[priority]; break; case MLX5_FLOW_NAMESPACE_EGRESS: prio = &dev->flow_db->egress_prios[priority]; break; case MLX5_FLOW_NAMESPACE_FDB: prio = &dev->flow_db->fdb; break; case MLX5_FLOW_NAMESPACE_RDMA_RX: prio = &dev->flow_db->rdma_rx[priority]; break; case MLX5_FLOW_NAMESPACE_RDMA_TX: prio = &dev->flow_db->rdma_tx[priority]; break; default: return ERR_PTR(-EINVAL); } if (!prio) return ERR_PTR(-EINVAL); if (prio->flow_table) return prio; return _get_prio(ns, prio, priority, max_table_size, MLX5_FS_MAX_TYPES, flags); } static struct mlx5_ib_flow_handler * _create_raw_flow_rule(struct mlx5_ib_dev *dev, struct mlx5_ib_flow_prio *ft_prio, struct mlx5_flow_destination *dst, struct mlx5_ib_flow_matcher *fs_matcher, struct mlx5_flow_context *flow_context, struct mlx5_flow_act *flow_act, void *cmd_in, int inlen, int dst_num) { struct mlx5_ib_flow_handler *handler; struct mlx5_flow_spec *spec; struct mlx5_flow_table *ft = ft_prio->flow_table; int err = 0; spec = kvzalloc(sizeof(*spec), GFP_KERNEL); handler = kzalloc(sizeof(*handler), GFP_KERNEL); if (!handler || !spec) { err = -ENOMEM; goto free; } INIT_LIST_HEAD(&handler->list); memcpy(spec->match_value, cmd_in, inlen); memcpy(spec->match_criteria, fs_matcher->matcher_mask.match_params, fs_matcher->mask_len); spec->match_criteria_enable = fs_matcher->match_criteria_enable; spec->flow_context = *flow_context; handler->rule = mlx5_add_flow_rules(ft, spec, flow_act, dst, dst_num); if (IS_ERR(handler->rule)) { err = PTR_ERR(handler->rule); goto free; } ft_prio->refcount++; handler->prio = ft_prio; handler->dev = dev; ft_prio->flow_table = ft; free: if (err) kfree(handler); kvfree(spec); return err ? ERR_PTR(err) : handler; } static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher *fs_matcher, void *match_v) { void *match_c; void *match_v_set_lyr_2_4, *match_c_set_lyr_2_4; void *dmac, *dmac_mask; void *ipv4, *ipv4_mask; if (!(fs_matcher->match_criteria_enable & (1 << MATCH_CRITERIA_ENABLE_OUTER_BIT))) return false; match_c = fs_matcher->matcher_mask.match_params; match_v_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_v, outer_headers); match_c_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_c, outer_headers); dmac = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4, dmac_47_16); dmac_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4, dmac_47_16); if (is_multicast_ether_addr(dmac) && is_multicast_ether_addr(dmac_mask)) return true; ipv4 = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4, dst_ipv4_dst_ipv6.ipv4_layout.ipv4); ipv4_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4, dst_ipv4_dst_ipv6.ipv4_layout.ipv4); if (ipv4_is_multicast(*(__be32 *)(ipv4)) && ipv4_is_multicast(*(__be32 *)(ipv4_mask))) return true; return false; } static struct mlx5_ib_flow_handler *raw_fs_rule_add( struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher, struct mlx5_flow_context *flow_context, struct mlx5_flow_act *flow_act, u32 counter_id, void *cmd_in, int inlen, int dest_id, int dest_type) { struct mlx5_flow_destination *dst; struct mlx5_ib_flow_prio *ft_prio; struct mlx5_ib_flow_handler *handler; int dst_num = 0; bool mcast; int err; if (fs_matcher->flow_type != MLX5_IB_FLOW_TYPE_NORMAL) return ERR_PTR(-EOPNOTSUPP); if (fs_matcher->priority > MLX5_IB_FLOW_LAST_PRIO) return ERR_PTR(-ENOMEM); dst = kcalloc(2, sizeof(*dst), GFP_KERNEL); if (!dst) return ERR_PTR(-ENOMEM); mcast = raw_fs_is_multicast(fs_matcher, cmd_in); mutex_lock(&dev->flow_db->lock); ft_prio = _get_flow_table(dev, fs_matcher, mcast); if (IS_ERR(ft_prio)) { err = PTR_ERR(ft_prio); goto unlock; } switch (dest_type) { case MLX5_FLOW_DESTINATION_TYPE_TIR: dst[dst_num].type = dest_type; dst[dst_num++].tir_num = dest_id; flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; break; case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE: dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM; dst[dst_num++].ft_num = dest_id; flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; break; case MLX5_FLOW_DESTINATION_TYPE_PORT: dst[dst_num++].type = MLX5_FLOW_DESTINATION_TYPE_PORT; flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW; break; default: break; } if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; dst[dst_num].counter_id = counter_id; dst_num++; } handler = _create_raw_flow_rule(dev, ft_prio, dst_num ? dst : NULL, fs_matcher, flow_context, flow_act, cmd_in, inlen, dst_num); if (IS_ERR(handler)) { err = PTR_ERR(handler); goto destroy_ft; } mutex_unlock(&dev->flow_db->lock); atomic_inc(&fs_matcher->usecnt); handler->flow_matcher = fs_matcher; kfree(dst); return handler; destroy_ft: put_flow_table(dev, ft_prio, false); unlock: mutex_unlock(&dev->flow_db->lock); kfree(dst); return ERR_PTR(err); } static u32 mlx5_ib_flow_action_flags_to_accel_xfrm_flags(u32 mlx5_flags) { u32 flags = 0; if (mlx5_flags & MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA) flags |= MLX5_ACCEL_XFRM_FLAG_REQUIRE_METADATA; return flags; } #define MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED \ MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA static struct ib_flow_action * mlx5_ib_create_flow_action_esp(struct ib_device *device, const struct ib_flow_action_attrs_esp *attr, struct uverbs_attr_bundle *attrs) { struct mlx5_ib_dev *mdev = to_mdev(device); struct ib_uverbs_flow_action_esp_keymat_aes_gcm *aes_gcm; struct mlx5_accel_esp_xfrm_attrs accel_attrs = {}; struct mlx5_ib_flow_action *action; u64 action_flags; u64 flags; int err = 0; err = uverbs_get_flags64( &action_flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_ACTION_FLAGS, ((MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED << 1) - 1)); if (err) return ERR_PTR(err); flags = mlx5_ib_flow_action_flags_to_accel_xfrm_flags(action_flags); /* We current only support a subset of the standard features. Only a * keymat of type AES_GCM, with icv_len == 16, iv_algo == SEQ and esn * (with overlap). Full offload mode isn't supported. */ if (!attr->keymat || attr->replay || attr->encap || attr->spi || attr->seq || attr->tfc_pad || attr->hard_limit_pkts || (attr->flags & ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED | IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT))) return ERR_PTR(-EOPNOTSUPP); if (attr->keymat->protocol != IB_UVERBS_FLOW_ACTION_ESP_KEYMAT_AES_GCM) return ERR_PTR(-EOPNOTSUPP); aes_gcm = &attr->keymat->keymat.aes_gcm; if (aes_gcm->icv_len != 16 || aes_gcm->iv_algo != IB_UVERBS_FLOW_ACTION_IV_ALGO_SEQ) return ERR_PTR(-EOPNOTSUPP); action = kmalloc(sizeof(*action), GFP_KERNEL); if (!action) return ERR_PTR(-ENOMEM); action->esp_aes_gcm.ib_flags = attr->flags; memcpy(&accel_attrs.keymat.aes_gcm.aes_key, &aes_gcm->aes_key, sizeof(accel_attrs.keymat.aes_gcm.aes_key)); accel_attrs.keymat.aes_gcm.key_len = aes_gcm->key_len * 8; memcpy(&accel_attrs.keymat.aes_gcm.salt, &aes_gcm->salt, sizeof(accel_attrs.keymat.aes_gcm.salt)); memcpy(&accel_attrs.keymat.aes_gcm.seq_iv, &aes_gcm->iv, sizeof(accel_attrs.keymat.aes_gcm.seq_iv)); accel_attrs.keymat.aes_gcm.icv_len = aes_gcm->icv_len * 8; accel_attrs.keymat.aes_gcm.iv_algo = MLX5_ACCEL_ESP_AES_GCM_IV_ALGO_SEQ; accel_attrs.keymat_type = MLX5_ACCEL_ESP_KEYMAT_AES_GCM; accel_attrs.esn = attr->esn; if (attr->flags & IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED) accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED; if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW) accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP; if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT) accel_attrs.action |= MLX5_ACCEL_ESP_ACTION_ENCRYPT; action->esp_aes_gcm.ctx = mlx5_accel_esp_create_xfrm(mdev->mdev, &accel_attrs, flags); if (IS_ERR(action->esp_aes_gcm.ctx)) { err = PTR_ERR(action->esp_aes_gcm.ctx); goto err_parse; } action->esp_aes_gcm.ib_flags = attr->flags; return &action->ib_action; err_parse: kfree(action); return ERR_PTR(err); } static int mlx5_ib_modify_flow_action_esp(struct ib_flow_action *action, const struct ib_flow_action_attrs_esp *attr, struct uverbs_attr_bundle *attrs) { struct mlx5_ib_flow_action *maction = to_mflow_act(action); struct mlx5_accel_esp_xfrm_attrs accel_attrs; int err = 0; if (attr->keymat || attr->replay || attr->encap || attr->spi || attr->seq || attr->tfc_pad || attr->hard_limit_pkts || (attr->flags & ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED | IB_FLOW_ACTION_ESP_FLAGS_MOD_ESP_ATTRS | IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW))) return -EOPNOTSUPP; /* Only the ESN value or the MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP can * be modified. */ if (!(maction->esp_aes_gcm.ib_flags & IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED) && attr->flags & (IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED | IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)) return -EINVAL; memcpy(&accel_attrs, &maction->esp_aes_gcm.ctx->attrs, sizeof(accel_attrs)); accel_attrs.esn = attr->esn; if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW) accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP; else accel_attrs.flags &= ~MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP; err = mlx5_accel_esp_modify_xfrm(maction->esp_aes_gcm.ctx, &accel_attrs); if (err) return err; maction->esp_aes_gcm.ib_flags &= ~IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW; maction->esp_aes_gcm.ib_flags |= attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW; return 0; } static void destroy_flow_action_raw(struct mlx5_ib_flow_action *maction) { switch (maction->flow_action_raw.sub_type) { case MLX5_IB_FLOW_ACTION_MODIFY_HEADER: mlx5_modify_header_dealloc(maction->flow_action_raw.dev->mdev, maction->flow_action_raw.modify_hdr); break; case MLX5_IB_FLOW_ACTION_PACKET_REFORMAT: mlx5_packet_reformat_dealloc(maction->flow_action_raw.dev->mdev, maction->flow_action_raw.pkt_reformat); break; case MLX5_IB_FLOW_ACTION_DECAP: break; default: break; } } static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action) { struct mlx5_ib_flow_action *maction = to_mflow_act(action); switch (action->type) { case IB_FLOW_ACTION_ESP: /* * We only support aes_gcm by now, so we implicitly know this is * the underline crypto. */ mlx5_accel_esp_destroy_xfrm(maction->esp_aes_gcm.ctx); break; case IB_FLOW_ACTION_UNSPECIFIED: destroy_flow_action_raw(maction); break; default: WARN_ON(true); break; } kfree(maction); return 0; } static int mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type, enum mlx5_flow_namespace_type *namespace) { switch (table_type) { case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX: *namespace = MLX5_FLOW_NAMESPACE_BYPASS; break; case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX: *namespace = MLX5_FLOW_NAMESPACE_EGRESS; break; case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB: *namespace = MLX5_FLOW_NAMESPACE_FDB; break; case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX: *namespace = MLX5_FLOW_NAMESPACE_RDMA_RX; break; case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TX: *namespace = MLX5_FLOW_NAMESPACE_RDMA_TX; break; default: return -EINVAL; } return 0; } static const struct uverbs_attr_spec mlx5_ib_flow_type[] = { [MLX5_IB_FLOW_TYPE_NORMAL] = { .type = UVERBS_ATTR_TYPE_PTR_IN, .u.ptr = { .len = sizeof(u16), /* data is priority */ .min_len = sizeof(u16), } }, [MLX5_IB_FLOW_TYPE_SNIFFER] = { .type = UVERBS_ATTR_TYPE_PTR_IN, UVERBS_ATTR_NO_DATA(), }, [MLX5_IB_FLOW_TYPE_ALL_DEFAULT] = { .type = UVERBS_ATTR_TYPE_PTR_IN, UVERBS_ATTR_NO_DATA(), }, [MLX5_IB_FLOW_TYPE_MC_DEFAULT] = { .type = UVERBS_ATTR_TYPE_PTR_IN, UVERBS_ATTR_NO_DATA(), }, }; static bool is_flow_dest(void *obj, int *dest_id, int *dest_type) { struct devx_obj *devx_obj = obj; u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); switch (opcode) { case MLX5_CMD_OP_DESTROY_TIR: *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, obj_id); return true; case MLX5_CMD_OP_DESTROY_FLOW_TABLE: *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox, table_id); return true; default: return false; } } static int get_dests(struct uverbs_attr_bundle *attrs, struct mlx5_ib_flow_matcher *fs_matcher, int *dest_id, int *dest_type, struct ib_qp **qp, u32 *flags) { bool dest_devx, dest_qp; void *devx_obj; int err; dest_devx = uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX); dest_qp = uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_CREATE_FLOW_DEST_QP); *flags = 0; err = uverbs_get_flags32(flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_FLAGS, MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS | MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP); if (err) return err; /* Both flags are not allowed */ if (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS && *flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP) return -EINVAL; if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) { if (dest_devx && (dest_qp || *flags)) return -EINVAL; else if (dest_qp && *flags) return -EINVAL; } /* Allow only DEVX object, drop as dest for FDB */ if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB && !(dest_devx || (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP))) return -EINVAL; /* Allow only DEVX object or QP as dest when inserting to RDMA_RX */ if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) && ((!dest_devx && !dest_qp) || (dest_devx && dest_qp))) return -EINVAL; *qp = NULL; if (dest_devx) { devx_obj = uverbs_attr_get_obj(attrs, MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX); /* Verify that the given DEVX object is a flow * steering destination. */ if (!is_flow_dest(devx_obj, dest_id, dest_type)) return -EINVAL; /* Allow only flow table as dest when inserting to FDB or RDMA_RX */ if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB || fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) && *dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) return -EINVAL; } else if (dest_qp) { struct mlx5_ib_qp *mqp; *qp = uverbs_attr_get_obj(attrs, MLX5_IB_ATTR_CREATE_FLOW_DEST_QP); if (IS_ERR(*qp)) return PTR_ERR(*qp); if ((*qp)->qp_type != IB_QPT_RAW_PACKET) return -EINVAL; mqp = to_mqp(*qp); if (mqp->is_rss) *dest_id = mqp->rss_qp.tirn; else *dest_id = mqp->raw_packet_qp.rq.tirn; *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; } else if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS || fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) && !(*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)) { *dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT; } if (*dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR && (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS || fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX)) return -EINVAL; return 0; } static bool is_flow_counter(void *obj, u32 offset, u32 *counter_id) { struct devx_obj *devx_obj = obj; u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) { if (offset && offset >= devx_obj->flow_counter_bulk_size) return false; *counter_id = MLX5_GET(dealloc_flow_counter_in, devx_obj->dinbox, flow_counter_id); *counter_id += offset; return true; } return false; } #define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2 static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)( struct uverbs_attr_bundle *attrs) { struct mlx5_flow_context flow_context = {.flow_tag = MLX5_FS_DEFAULT_FLOW_TAG}; u32 *offset_attr, offset = 0, counter_id = 0; int dest_id, dest_type = -1, inlen, len, ret, i; struct mlx5_ib_flow_handler *flow_handler; struct mlx5_ib_flow_matcher *fs_matcher; struct ib_uobject **arr_flow_actions; struct ib_uflow_resources *uflow_res; struct mlx5_flow_act flow_act = {}; struct ib_qp *qp = NULL; void *devx_obj, *cmd_in; struct ib_uobject *uobj; struct mlx5_ib_dev *dev; u32 flags; if (!capable(CAP_NET_RAW)) return -EPERM; fs_matcher = uverbs_attr_get_obj(attrs, MLX5_IB_ATTR_CREATE_FLOW_MATCHER); uobj = uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_FLOW_HANDLE); dev = mlx5_udata_to_mdev(&attrs->driver_udata); if (get_dests(attrs, fs_matcher, &dest_id, &dest_type, &qp, &flags)) return -EINVAL; if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS) flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS; if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP) flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP; len = uverbs_attr_get_uobjs_arr(attrs, MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, &arr_flow_actions); if (len) { devx_obj = arr_flow_actions[0]->object; if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET)) { int num_offsets = uverbs_attr_ptr_get_array_size( attrs, MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET, sizeof(u32)); if (num_offsets != 1) return -EINVAL; offset_attr = uverbs_attr_get_alloced_ptr( attrs, MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET); offset = *offset_attr; } if (!is_flow_counter(devx_obj, offset, &counter_id)) return -EINVAL; flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; } cmd_in = uverbs_attr_get_alloced_ptr( attrs, MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE); inlen = uverbs_attr_get_len(attrs, MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE); uflow_res = flow_resources_alloc(MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS); if (!uflow_res) return -ENOMEM; len = uverbs_attr_get_uobjs_arr(attrs, MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, &arr_flow_actions); for (i = 0; i < len; i++) { struct mlx5_ib_flow_action *maction = to_mflow_act(arr_flow_actions[i]->object); ret = parse_flow_flow_action(maction, false, &flow_act); if (ret) goto err_out; flow_resources_add(uflow_res, IB_FLOW_SPEC_ACTION_HANDLE, arr_flow_actions[i]->object); } ret = uverbs_copy_from(&flow_context.flow_tag, attrs, MLX5_IB_ATTR_CREATE_FLOW_TAG); if (!ret) { if (flow_context.flow_tag >= BIT(24)) { ret = -EINVAL; goto err_out; } flow_context.flags |= FLOW_CONTEXT_HAS_TAG; } flow_handler = raw_fs_rule_add(dev, fs_matcher, &flow_context, &flow_act, counter_id, cmd_in, inlen, dest_id, dest_type); if (IS_ERR(flow_handler)) { ret = PTR_ERR(flow_handler); goto err_out; } ib_set_flow(uobj, &flow_handler->ibflow, qp, &dev->ib_dev, uflow_res); return 0; err_out: ib_uverbs_flow_resources_free(uflow_res); return ret; } static int flow_matcher_cleanup(struct ib_uobject *uobject, enum rdma_remove_reason why, struct uverbs_attr_bundle *attrs) { struct mlx5_ib_flow_matcher *obj = uobject->object; if (atomic_read(&obj->usecnt)) return -EBUSY; kfree(obj); return 0; } static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle *attrs, struct mlx5_ib_flow_matcher *obj) { enum mlx5_ib_uapi_flow_table_type ft_type = MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX; u32 flags; int err; /* New users should use MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE and older * users should switch to it. We leave this to not break userspace */ if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE) && uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) return -EINVAL; if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE)) { err = uverbs_get_const(&ft_type, attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE); if (err) return err; err = mlx5_ib_ft_type_to_namespace(ft_type, &obj->ns_type); if (err) return err; return 0; } if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) { err = uverbs_get_flags32(&flags, attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS, IB_FLOW_ATTR_FLAGS_EGRESS); if (err) return err; if (flags) { mlx5_ib_ft_type_to_namespace( MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX, &obj->ns_type); return 0; } } obj->ns_type = MLX5_FLOW_NAMESPACE_BYPASS; return 0; } static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)( struct uverbs_attr_bundle *attrs) { struct ib_uobject *uobj = uverbs_attr_get_uobject( attrs, MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE); struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata); struct mlx5_ib_flow_matcher *obj; int err; obj = kzalloc(sizeof(struct mlx5_ib_flow_matcher), GFP_KERNEL); if (!obj) return -ENOMEM; obj->mask_len = uverbs_attr_get_len( attrs, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK); err = uverbs_copy_from(&obj->matcher_mask, attrs, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK); if (err) goto end; obj->flow_type = uverbs_attr_get_enum_id( attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE); if (obj->flow_type == MLX5_IB_FLOW_TYPE_NORMAL) { err = uverbs_copy_from(&obj->priority, attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE); if (err) goto end; } err = uverbs_copy_from(&obj->match_criteria_enable, attrs, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA); if (err) goto end; err = mlx5_ib_matcher_ns(attrs, obj); if (err) goto end; if (obj->ns_type == MLX5_FLOW_NAMESPACE_FDB && mlx5_eswitch_mode(dev->mdev) != MLX5_ESWITCH_OFFLOADS) { err = -EINVAL; goto end; } uobj->object = obj; obj->mdev = dev->mdev; atomic_set(&obj->usecnt, 0); return 0; end: kfree(obj); return err; } static struct ib_flow_action * mlx5_ib_create_modify_header(struct mlx5_ib_dev *dev, enum mlx5_ib_uapi_flow_table_type ft_type, u8 num_actions, void *in) { enum mlx5_flow_namespace_type namespace; struct mlx5_ib_flow_action *maction; int ret; ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace); if (ret) return ERR_PTR(-EINVAL); maction = kzalloc(sizeof(*maction), GFP_KERNEL); if (!maction) return ERR_PTR(-ENOMEM); maction->flow_action_raw.modify_hdr = mlx5_modify_header_alloc(dev->mdev, namespace, num_actions, in); if (IS_ERR(maction->flow_action_raw.modify_hdr)) { ret = PTR_ERR(maction->flow_action_raw.modify_hdr); kfree(maction); return ERR_PTR(ret); } maction->flow_action_raw.sub_type = MLX5_IB_FLOW_ACTION_MODIFY_HEADER; maction->flow_action_raw.dev = dev; return &maction->ib_action; } static bool mlx5_ib_modify_header_supported(struct mlx5_ib_dev *dev) { return MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, max_modify_header_actions) || MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, max_modify_header_actions) || MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, max_modify_header_actions); } static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)( struct uverbs_attr_bundle *attrs) { struct ib_uobject *uobj = uverbs_attr_get_uobject( attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE); struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata); enum mlx5_ib_uapi_flow_table_type ft_type; struct ib_flow_action *action; int num_actions; void *in; int ret; if (!mlx5_ib_modify_header_supported(mdev)) return -EOPNOTSUPP; in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM); num_actions = uverbs_attr_ptr_get_array_size( attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM, MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)); if (num_actions < 0) return num_actions; ret = uverbs_get_const(&ft_type, attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE); if (ret) return ret; action = mlx5_ib_create_modify_header(mdev, ft_type, num_actions, in); if (IS_ERR(action)) return PTR_ERR(action); uverbs_flow_action_fill_action(action, uobj, &mdev->ib_dev, IB_FLOW_ACTION_UNSPECIFIED); return 0; } static bool mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev *ibdev, u8 packet_reformat_type, u8 ft_type) { switch (packet_reformat_type) { case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL: if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX) return MLX5_CAP_FLOWTABLE(ibdev->mdev, encap_general_header); break; case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL: if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX) return MLX5_CAP_FLOWTABLE_NIC_TX(ibdev->mdev, reformat_l2_to_l3_tunnel); break; case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2: if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX) return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, reformat_l3_tunnel_to_l2); break; case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2: if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX) return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, decap); break; default: break; } return false; } static int mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt, u8 *prm_prt) { switch (dv_prt) { case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL: *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL; break; case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2: *prm_prt = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2; break; case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL: *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL; break; default: return -EINVAL; } return 0; } static int mlx5_ib_flow_action_create_packet_reformat_ctx( struct mlx5_ib_dev *dev, struct mlx5_ib_flow_action *maction, u8 ft_type, u8 dv_prt, void *in, size_t len) { struct mlx5_pkt_reformat_params reformat_params; enum mlx5_flow_namespace_type namespace; u8 prm_prt; int ret; ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace); if (ret) return ret; ret = mlx5_ib_dv_to_prm_packet_reforamt_type(dv_prt, &prm_prt); if (ret) return ret; memset(&reformat_params, 0, sizeof(reformat_params)); reformat_params.type = prm_prt; reformat_params.size = len; reformat_params.data = in; maction->flow_action_raw.pkt_reformat = mlx5_packet_reformat_alloc(dev->mdev, &reformat_params, namespace); if (IS_ERR(maction->flow_action_raw.pkt_reformat)) { ret = PTR_ERR(maction->flow_action_raw.pkt_reformat); return ret; } maction->flow_action_raw.sub_type = MLX5_IB_FLOW_ACTION_PACKET_REFORMAT; maction->flow_action_raw.dev = dev; return 0; } static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)( struct uverbs_attr_bundle *attrs) { struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE); struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata); enum mlx5_ib_uapi_flow_action_packet_reformat_type dv_prt; enum mlx5_ib_uapi_flow_table_type ft_type; struct mlx5_ib_flow_action *maction; int ret; ret = uverbs_get_const(&ft_type, attrs, MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE); if (ret) return ret; ret = uverbs_get_const(&dv_prt, attrs, MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE); if (ret) return ret; if (!mlx5_ib_flow_action_packet_reformat_valid(mdev, dv_prt, ft_type)) return -EOPNOTSUPP; maction = kzalloc(sizeof(*maction), GFP_KERNEL); if (!maction) return -ENOMEM; if (dv_prt == MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2) { maction->flow_action_raw.sub_type = MLX5_IB_FLOW_ACTION_DECAP; maction->flow_action_raw.dev = mdev; } else { void *in; int len; in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF); if (IS_ERR(in)) { ret = PTR_ERR(in); goto free_maction; } len = uverbs_attr_get_len(attrs, MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF); ret = mlx5_ib_flow_action_create_packet_reformat_ctx(mdev, maction, ft_type, dv_prt, in, len); if (ret) goto free_maction; } uverbs_flow_action_fill_action(&maction->ib_action, uobj, &mdev->ib_dev, IB_FLOW_ACTION_UNSPECIFIED); return 0; free_maction: kfree(maction); return ret; } DECLARE_UVERBS_NAMED_METHOD( MLX5_IB_METHOD_CREATE_FLOW, UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE, UVERBS_OBJECT_FLOW, UVERBS_ACCESS_NEW, UA_MANDATORY), UVERBS_ATTR_PTR_IN( MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE, UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)), UA_MANDATORY, UA_ALLOC_AND_COPY), UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_MATCHER, MLX5_IB_OBJECT_FLOW_MATCHER, UVERBS_ACCESS_READ, UA_MANDATORY), UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_QP, UVERBS_OBJECT_QP, UVERBS_ACCESS_READ), UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX, MLX5_IB_OBJECT_DEVX_OBJ, UVERBS_ACCESS_READ), UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, UVERBS_OBJECT_FLOW_ACTION, UVERBS_ACCESS_READ, 1, MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS, UA_OPTIONAL), UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_TAG, UVERBS_ATTR_TYPE(u32), UA_OPTIONAL), UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, MLX5_IB_OBJECT_DEVX_OBJ, UVERBS_ACCESS_READ, 1, 1, UA_OPTIONAL), UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET, UVERBS_ATTR_MIN_SIZE(sizeof(u32)), UA_OPTIONAL, UA_ALLOC_AND_COPY), UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_FLAGS, enum mlx5_ib_create_flow_flags, UA_OPTIONAL)); DECLARE_UVERBS_NAMED_METHOD_DESTROY( MLX5_IB_METHOD_DESTROY_FLOW, UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE, UVERBS_OBJECT_FLOW, UVERBS_ACCESS_DESTROY, UA_MANDATORY)); ADD_UVERBS_METHODS(mlx5_ib_fs, UVERBS_OBJECT_FLOW, &UVERBS_METHOD(MLX5_IB_METHOD_CREATE_FLOW), &UVERBS_METHOD(MLX5_IB_METHOD_DESTROY_FLOW)); DECLARE_UVERBS_NAMED_METHOD( MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER, UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE, UVERBS_OBJECT_FLOW_ACTION, UVERBS_ACCESS_NEW, UA_MANDATORY), UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM, UVERBS_ATTR_MIN_SIZE(MLX5_UN_SZ_BYTES( set_add_copy_action_in_auto)), UA_MANDATORY, UA_ALLOC_AND_COPY), UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE, enum mlx5_ib_uapi_flow_table_type, UA_MANDATORY)); DECLARE_UVERBS_NAMED_METHOD( MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT, UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE, UVERBS_OBJECT_FLOW_ACTION, UVERBS_ACCESS_NEW, UA_MANDATORY), UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF, UVERBS_ATTR_MIN_SIZE(1), UA_ALLOC_AND_COPY, UA_OPTIONAL), UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE, enum mlx5_ib_uapi_flow_action_packet_reformat_type, UA_MANDATORY), UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE, enum mlx5_ib_uapi_flow_table_type, UA_MANDATORY)); ADD_UVERBS_METHODS( mlx5_ib_flow_actions, UVERBS_OBJECT_FLOW_ACTION, &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER), &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)); DECLARE_UVERBS_NAMED_METHOD( MLX5_IB_METHOD_FLOW_MATCHER_CREATE, UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE, MLX5_IB_OBJECT_FLOW_MATCHER, UVERBS_ACCESS_NEW, UA_MANDATORY), UVERBS_ATTR_PTR_IN( MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK, UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)), UA_MANDATORY), UVERBS_ATTR_ENUM_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE, mlx5_ib_flow_type, UA_MANDATORY), UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA, UVERBS_ATTR_TYPE(u8), UA_MANDATORY), UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS, enum ib_flow_flags, UA_OPTIONAL), UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE, enum mlx5_ib_uapi_flow_table_type, UA_OPTIONAL)); DECLARE_UVERBS_NAMED_METHOD_DESTROY( MLX5_IB_METHOD_FLOW_MATCHER_DESTROY, UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE, MLX5_IB_OBJECT_FLOW_MATCHER, UVERBS_ACCESS_DESTROY, UA_MANDATORY)); DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_FLOW_MATCHER, UVERBS_TYPE_ALLOC_IDR(flow_matcher_cleanup), &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_CREATE), &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_DESTROY)); const struct uapi_definition mlx5_ib_flow_defs[] = { UAPI_DEF_CHAIN_OBJ_TREE_NAMED( MLX5_IB_OBJECT_FLOW_MATCHER), UAPI_DEF_CHAIN_OBJ_TREE( UVERBS_OBJECT_FLOW, &mlx5_ib_fs), UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION, &mlx5_ib_flow_actions), {}, }; static const struct ib_device_ops flow_ops = { .create_flow = mlx5_ib_create_flow, .destroy_flow = mlx5_ib_destroy_flow, .destroy_flow_action = mlx5_ib_destroy_flow_action, }; static const struct ib_device_ops flow_ipsec_ops = { .create_flow_action_esp = mlx5_ib_create_flow_action_esp, .modify_flow_action_esp = mlx5_ib_modify_flow_action_esp, }; int mlx5_ib_fs_init(struct mlx5_ib_dev *dev) { dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL); if (!dev->flow_db) return -ENOMEM; mutex_init(&dev->flow_db->lock); ib_set_device_ops(&dev->ib_dev, &flow_ops); if (mlx5_accel_ipsec_device_caps(dev->mdev) & MLX5_ACCEL_IPSEC_CAP_DEVICE) ib_set_device_ops(&dev->ib_dev, &flow_ipsec_ops); return 0; }
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1