Author | Tokens | Token Proportion | Commits | Commit Proportion |
---|---|---|---|---|
Edward Cree | 2941 | 93.60% | 15 | 71.43% |
Ben Hutchings | 200 | 6.37% | 5 | 23.81% |
Thomas Gleixner | 1 | 0.03% | 1 | 4.76% |
Total | 3142 | 21 |
// SPDX-License-Identifier: GPL-2.0-only /**************************************************************************** * Driver for Solarflare network controllers and boards * Copyright 2023, Advanced Micro Devices, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published * by the Free Software Foundation, incorporated herein by reference. */ #include "tc_conntrack.h" #include "tc.h" #include "mae.h" static int efx_tc_flow_block(enum tc_setup_type type, void *type_data, void *cb_priv); static const struct rhashtable_params efx_tc_ct_zone_ht_params = { .key_len = offsetof(struct efx_tc_ct_zone, linkage), .key_offset = 0, .head_offset = offsetof(struct efx_tc_ct_zone, linkage), }; static const struct rhashtable_params efx_tc_ct_ht_params = { .key_len = offsetof(struct efx_tc_ct_entry, linkage), .key_offset = 0, .head_offset = offsetof(struct efx_tc_ct_entry, linkage), }; static void efx_tc_ct_zone_free(void *ptr, void *arg) { struct efx_tc_ct_zone *zone = ptr; struct efx_nic *efx = zone->efx; netif_err(efx, drv, efx->net_dev, "tc ct_zone %u still present at teardown, removing\n", zone->zone); nf_flow_table_offload_del_cb(zone->nf_ft, efx_tc_flow_block, zone); kfree(zone); } static void efx_tc_ct_free(void *ptr, void *arg) { struct efx_tc_ct_entry *conn = ptr; struct efx_nic *efx = arg; netif_err(efx, drv, efx->net_dev, "tc ct_entry %lx still present at teardown\n", conn->cookie); /* We can release the counter, but we can't remove the CT itself * from hardware because the table meta is already gone. */ efx_tc_flower_release_counter(efx, conn->cnt); kfree(conn); } int efx_tc_init_conntrack(struct efx_nic *efx) { int rc; rc = rhashtable_init(&efx->tc->ct_zone_ht, &efx_tc_ct_zone_ht_params); if (rc < 0) goto fail_ct_zone_ht; rc = rhashtable_init(&efx->tc->ct_ht, &efx_tc_ct_ht_params); if (rc < 0) goto fail_ct_ht; return 0; fail_ct_ht: rhashtable_destroy(&efx->tc->ct_zone_ht); fail_ct_zone_ht: return rc; } /* Only call this in init failure teardown. * Normal exit should fini instead as there may be entries in the table. */ void efx_tc_destroy_conntrack(struct efx_nic *efx) { rhashtable_destroy(&efx->tc->ct_ht); rhashtable_destroy(&efx->tc->ct_zone_ht); } void efx_tc_fini_conntrack(struct efx_nic *efx) { rhashtable_free_and_destroy(&efx->tc->ct_zone_ht, efx_tc_ct_zone_free, NULL); rhashtable_free_and_destroy(&efx->tc->ct_ht, efx_tc_ct_free, efx); } #define EFX_NF_TCP_FLAG(flg) cpu_to_be16(be32_to_cpu(TCP_FLAG_##flg) >> 16) static int efx_tc_ct_parse_match(struct efx_nic *efx, struct flow_rule *fr, struct efx_tc_ct_entry *conn) { struct flow_dissector *dissector = fr->match.dissector; unsigned char ipv = 0; bool tcp = false; if (flow_rule_match_key(fr, FLOW_DISSECTOR_KEY_CONTROL)) { struct flow_match_control fm; flow_rule_match_control(fr, &fm); if (IS_ALL_ONES(fm.mask->addr_type)) switch (fm.key->addr_type) { case FLOW_DISSECTOR_KEY_IPV4_ADDRS: ipv = 4; break; case FLOW_DISSECTOR_KEY_IPV6_ADDRS: ipv = 6; break; default: break; } } if (!ipv) { netif_dbg(efx, drv, efx->net_dev, "Conntrack missing ipv specification\n"); return -EOPNOTSUPP; } if (dissector->used_keys & ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | BIT_ULL(FLOW_DISSECTOR_KEY_TCP) | BIT_ULL(FLOW_DISSECTOR_KEY_META))) { netif_dbg(efx, drv, efx->net_dev, "Unsupported conntrack keys %#llx\n", dissector->used_keys); return -EOPNOTSUPP; } if (flow_rule_match_key(fr, FLOW_DISSECTOR_KEY_BASIC)) { struct flow_match_basic fm; flow_rule_match_basic(fr, &fm); if (!IS_ALL_ONES(fm.mask->n_proto)) { netif_dbg(efx, drv, efx->net_dev, "Conntrack eth_proto is not exact-match; mask %04x\n", ntohs(fm.mask->n_proto)); return -EOPNOTSUPP; } conn->eth_proto = fm.key->n_proto; if (conn->eth_proto != (ipv == 4 ? htons(ETH_P_IP) : htons(ETH_P_IPV6))) { netif_dbg(efx, drv, efx->net_dev, "Conntrack eth_proto is not IPv%u, is %04x\n", ipv, ntohs(conn->eth_proto)); return -EOPNOTSUPP; } if (!IS_ALL_ONES(fm.mask->ip_proto)) { netif_dbg(efx, drv, efx->net_dev, "Conntrack ip_proto is not exact-match; mask %02x\n", fm.mask->ip_proto); return -EOPNOTSUPP; } conn->ip_proto = fm.key->ip_proto; switch (conn->ip_proto) { case IPPROTO_TCP: tcp = true; break; case IPPROTO_UDP: break; default: netif_dbg(efx, drv, efx->net_dev, "Conntrack ip_proto not TCP or UDP, is %02x\n", conn->ip_proto); return -EOPNOTSUPP; } } else { netif_dbg(efx, drv, efx->net_dev, "Conntrack missing eth_proto, ip_proto\n"); return -EOPNOTSUPP; } if (ipv == 4 && flow_rule_match_key(fr, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) { struct flow_match_ipv4_addrs fm; flow_rule_match_ipv4_addrs(fr, &fm); if (!IS_ALL_ONES(fm.mask->src)) { netif_dbg(efx, drv, efx->net_dev, "Conntrack ipv4.src is not exact-match; mask %08x\n", ntohl(fm.mask->src)); return -EOPNOTSUPP; } conn->src_ip = fm.key->src; if (!IS_ALL_ONES(fm.mask->dst)) { netif_dbg(efx, drv, efx->net_dev, "Conntrack ipv4.dst is not exact-match; mask %08x\n", ntohl(fm.mask->dst)); return -EOPNOTSUPP; } conn->dst_ip = fm.key->dst; } else if (ipv == 6 && flow_rule_match_key(fr, FLOW_DISSECTOR_KEY_IPV6_ADDRS)) { struct flow_match_ipv6_addrs fm; flow_rule_match_ipv6_addrs(fr, &fm); if (!efx_ipv6_addr_all_ones(&fm.mask->src)) { netif_dbg(efx, drv, efx->net_dev, "Conntrack ipv6.src is not exact-match; mask %pI6\n", &fm.mask->src); return -EOPNOTSUPP; } conn->src_ip6 = fm.key->src; if (!efx_ipv6_addr_all_ones(&fm.mask->dst)) { netif_dbg(efx, drv, efx->net_dev, "Conntrack ipv6.dst is not exact-match; mask %pI6\n", &fm.mask->dst); return -EOPNOTSUPP; } conn->dst_ip6 = fm.key->dst; } else { netif_dbg(efx, drv, efx->net_dev, "Conntrack missing IPv%u addrs\n", ipv); return -EOPNOTSUPP; } if (flow_rule_match_key(fr, FLOW_DISSECTOR_KEY_PORTS)) { struct flow_match_ports fm; flow_rule_match_ports(fr, &fm); if (!IS_ALL_ONES(fm.mask->src)) { netif_dbg(efx, drv, efx->net_dev, "Conntrack ports.src is not exact-match; mask %04x\n", ntohs(fm.mask->src)); return -EOPNOTSUPP; } conn->l4_sport = fm.key->src; if (!IS_ALL_ONES(fm.mask->dst)) { netif_dbg(efx, drv, efx->net_dev, "Conntrack ports.dst is not exact-match; mask %04x\n", ntohs(fm.mask->dst)); return -EOPNOTSUPP; } conn->l4_dport = fm.key->dst; } else { netif_dbg(efx, drv, efx->net_dev, "Conntrack missing L4 ports\n"); return -EOPNOTSUPP; } if (flow_rule_match_key(fr, FLOW_DISSECTOR_KEY_TCP)) { __be16 tcp_interesting_flags; struct flow_match_tcp fm; if (!tcp) { netif_dbg(efx, drv, efx->net_dev, "Conntrack matching on TCP keys but ipproto is not tcp\n"); return -EOPNOTSUPP; } flow_rule_match_tcp(fr, &fm); tcp_interesting_flags = EFX_NF_TCP_FLAG(SYN) | EFX_NF_TCP_FLAG(RST) | EFX_NF_TCP_FLAG(FIN); /* If any of the tcp_interesting_flags is set, we always * inhibit CT lookup in LHS (so SW can update CT table). */ if (fm.key->flags & tcp_interesting_flags) { netif_dbg(efx, drv, efx->net_dev, "Unsupported conntrack tcp.flags %04x/%04x\n", ntohs(fm.key->flags), ntohs(fm.mask->flags)); return -EOPNOTSUPP; } /* Other TCP flags cannot be filtered at CT */ if (fm.mask->flags & ~tcp_interesting_flags) { netif_dbg(efx, drv, efx->net_dev, "Unsupported conntrack tcp.flags %04x/%04x\n", ntohs(fm.key->flags), ntohs(fm.mask->flags)); return -EOPNOTSUPP; } } return 0; } /** * struct efx_tc_ct_mangler_state - tracks which fields have been pedited * * @ipv4: IP source or destination addr has been set * @tcpudp: TCP/UDP source or destination port has been set */ struct efx_tc_ct_mangler_state { u8 ipv4:1; u8 tcpudp:1; }; static int efx_tc_ct_mangle(struct efx_nic *efx, struct efx_tc_ct_entry *conn, const struct flow_action_entry *fa, struct efx_tc_ct_mangler_state *mung) { /* Is this the first mangle we've processed for this rule? */ bool first = !(mung->ipv4 || mung->tcpudp); bool dnat = false; switch (fa->mangle.htype) { case FLOW_ACT_MANGLE_HDR_TYPE_IP4: switch (fa->mangle.offset) { case offsetof(struct iphdr, daddr): dnat = true; fallthrough; case offsetof(struct iphdr, saddr): if (fa->mangle.mask) return -EOPNOTSUPP; conn->nat_ip = htonl(fa->mangle.val); mung->ipv4 = 1; break; default: return -EOPNOTSUPP; } break; case FLOW_ACT_MANGLE_HDR_TYPE_TCP: case FLOW_ACT_MANGLE_HDR_TYPE_UDP: /* Both struct tcphdr and struct udphdr start with * __be16 source; * __be16 dest; * so we can use the same code for both. */ switch (fa->mangle.offset) { case offsetof(struct tcphdr, dest): BUILD_BUG_ON(offsetof(struct tcphdr, dest) != offsetof(struct udphdr, dest)); dnat = true; fallthrough; case offsetof(struct tcphdr, source): BUILD_BUG_ON(offsetof(struct tcphdr, source) != offsetof(struct udphdr, source)); if (~fa->mangle.mask != 0xffff) return -EOPNOTSUPP; conn->l4_natport = htons(fa->mangle.val); mung->tcpudp = 1; break; default: return -EOPNOTSUPP; } break; default: return -EOPNOTSUPP; } /* first mangle tells us whether this is SNAT or DNAT; * subsequent mangles must match that */ if (first) conn->dnat = dnat; else if (conn->dnat != dnat) return -EOPNOTSUPP; return 0; } static int efx_tc_ct_replace(struct efx_tc_ct_zone *ct_zone, struct flow_cls_offload *tc) { struct flow_rule *fr = flow_cls_offload_flow_rule(tc); struct efx_tc_ct_mangler_state mung = {}; struct efx_tc_ct_entry *conn, *old; struct efx_nic *efx = ct_zone->efx; const struct flow_action_entry *fa; struct efx_tc_counter *cnt; int rc, i; if (WARN_ON(!efx->tc)) return -ENETDOWN; if (WARN_ON(!efx->tc->up)) return -ENETDOWN; conn = kzalloc(sizeof(*conn), GFP_USER); if (!conn) return -ENOMEM; conn->cookie = tc->cookie; old = rhashtable_lookup_get_insert_fast(&efx->tc->ct_ht, &conn->linkage, efx_tc_ct_ht_params); if (IS_ERR(old)) { rc = PTR_ERR(old); goto release; } else if (old) { netif_dbg(efx, drv, efx->net_dev, "Already offloaded conntrack (cookie %lx)\n", tc->cookie); rc = -EEXIST; goto release; } /* Parse match */ conn->zone = ct_zone; rc = efx_tc_ct_parse_match(efx, fr, conn); if (rc) goto release; /* Parse actions */ flow_action_for_each(i, fa, &fr->action) { switch (fa->id) { case FLOW_ACTION_CT_METADATA: conn->mark = fa->ct_metadata.mark; if (memchr_inv(fa->ct_metadata.labels, 0, sizeof(fa->ct_metadata.labels))) { netif_dbg(efx, drv, efx->net_dev, "Setting CT label not supported\n"); rc = -EOPNOTSUPP; goto release; } break; case FLOW_ACTION_MANGLE: if (conn->eth_proto != htons(ETH_P_IP)) { netif_dbg(efx, drv, efx->net_dev, "NAT only supported for IPv4\n"); rc = -EOPNOTSUPP; goto release; } rc = efx_tc_ct_mangle(efx, conn, fa, &mung); if (rc) goto release; break; default: netif_dbg(efx, drv, efx->net_dev, "Unhandled action %u for conntrack\n", fa->id); rc = -EOPNOTSUPP; goto release; } } /* fill in defaults for unmangled values */ if (!mung.ipv4) conn->nat_ip = conn->dnat ? conn->dst_ip : conn->src_ip; if (!mung.tcpudp) conn->l4_natport = conn->dnat ? conn->l4_dport : conn->l4_sport; cnt = efx_tc_flower_allocate_counter(efx, EFX_TC_COUNTER_TYPE_CT); if (IS_ERR(cnt)) { rc = PTR_ERR(cnt); goto release; } conn->cnt = cnt; rc = efx_mae_insert_ct(efx, conn); if (rc) { netif_dbg(efx, drv, efx->net_dev, "Failed to insert conntrack, %d\n", rc); goto release; } mutex_lock(&ct_zone->mutex); list_add_tail(&conn->list, &ct_zone->cts); mutex_unlock(&ct_zone->mutex); return 0; release: if (conn->cnt) efx_tc_flower_release_counter(efx, conn->cnt); if (!old) rhashtable_remove_fast(&efx->tc->ct_ht, &conn->linkage, efx_tc_ct_ht_params); kfree(conn); return rc; } /* Caller must follow with efx_tc_ct_remove_finish() after RCU grace period! */ static void efx_tc_ct_remove(struct efx_nic *efx, struct efx_tc_ct_entry *conn) { int rc; /* Remove it from HW */ rc = efx_mae_remove_ct(efx, conn); /* Delete it from SW */ rhashtable_remove_fast(&efx->tc->ct_ht, &conn->linkage, efx_tc_ct_ht_params); if (rc) { netif_err(efx, drv, efx->net_dev, "Failed to remove conntrack %lx from hw, rc %d\n", conn->cookie, rc); } else { netif_dbg(efx, drv, efx->net_dev, "Removed conntrack %lx\n", conn->cookie); } } static void efx_tc_ct_remove_finish(struct efx_nic *efx, struct efx_tc_ct_entry *conn) { /* Remove related CT counter. This is delayed after the conn object we * are working with has been successfully removed. This protects the * counter from being used-after-free inside efx_tc_ct_stats. */ efx_tc_flower_release_counter(efx, conn->cnt); kfree(conn); } static int efx_tc_ct_destroy(struct efx_tc_ct_zone *ct_zone, struct flow_cls_offload *tc) { struct efx_nic *efx = ct_zone->efx; struct efx_tc_ct_entry *conn; conn = rhashtable_lookup_fast(&efx->tc->ct_ht, &tc->cookie, efx_tc_ct_ht_params); if (!conn) { netif_warn(efx, drv, efx->net_dev, "Conntrack %lx not found to remove\n", tc->cookie); return -ENOENT; } mutex_lock(&ct_zone->mutex); list_del(&conn->list); efx_tc_ct_remove(efx, conn); mutex_unlock(&ct_zone->mutex); synchronize_rcu(); efx_tc_ct_remove_finish(efx, conn); return 0; } static int efx_tc_ct_stats(struct efx_tc_ct_zone *ct_zone, struct flow_cls_offload *tc) { struct efx_nic *efx = ct_zone->efx; struct efx_tc_ct_entry *conn; struct efx_tc_counter *cnt; rcu_read_lock(); conn = rhashtable_lookup_fast(&efx->tc->ct_ht, &tc->cookie, efx_tc_ct_ht_params); if (!conn) { netif_warn(efx, drv, efx->net_dev, "Conntrack %lx not found for stats\n", tc->cookie); rcu_read_unlock(); return -ENOENT; } cnt = conn->cnt; spin_lock_bh(&cnt->lock); /* Report only last use */ flow_stats_update(&tc->stats, 0, 0, 0, cnt->touched, FLOW_ACTION_HW_STATS_DELAYED); spin_unlock_bh(&cnt->lock); rcu_read_unlock(); return 0; } static int efx_tc_flow_block(enum tc_setup_type type, void *type_data, void *cb_priv) { struct flow_cls_offload *tcb = type_data; struct efx_tc_ct_zone *ct_zone = cb_priv; if (type != TC_SETUP_CLSFLOWER) return -EOPNOTSUPP; switch (tcb->command) { case FLOW_CLS_REPLACE: return efx_tc_ct_replace(ct_zone, tcb); case FLOW_CLS_DESTROY: return efx_tc_ct_destroy(ct_zone, tcb); case FLOW_CLS_STATS: return efx_tc_ct_stats(ct_zone, tcb); default: break; } return -EOPNOTSUPP; } struct efx_tc_ct_zone *efx_tc_ct_register_zone(struct efx_nic *efx, u16 zone, struct nf_flowtable *ct_ft) { struct efx_tc_ct_zone *ct_zone, *old; int rc; ct_zone = kzalloc(sizeof(*ct_zone), GFP_USER); if (!ct_zone) return ERR_PTR(-ENOMEM); ct_zone->zone = zone; old = rhashtable_lookup_get_insert_fast(&efx->tc->ct_zone_ht, &ct_zone->linkage, efx_tc_ct_zone_ht_params); if (old) { /* don't need our new entry */ kfree(ct_zone); if (IS_ERR(old)) /* oh dear, it's actually an error */ return ERR_CAST(old); if (!refcount_inc_not_zero(&old->ref)) return ERR_PTR(-EAGAIN); /* existing entry found */ WARN_ON_ONCE(old->nf_ft != ct_ft); netif_dbg(efx, drv, efx->net_dev, "Found existing ct_zone for %u\n", zone); return old; } ct_zone->nf_ft = ct_ft; ct_zone->efx = efx; INIT_LIST_HEAD(&ct_zone->cts); mutex_init(&ct_zone->mutex); rc = nf_flow_table_offload_add_cb(ct_ft, efx_tc_flow_block, ct_zone); netif_dbg(efx, drv, efx->net_dev, "Adding new ct_zone for %u, rc %d\n", zone, rc); if (rc < 0) goto fail; refcount_set(&ct_zone->ref, 1); return ct_zone; fail: rhashtable_remove_fast(&efx->tc->ct_zone_ht, &ct_zone->linkage, efx_tc_ct_zone_ht_params); kfree(ct_zone); return ERR_PTR(rc); } void efx_tc_ct_unregister_zone(struct efx_nic *efx, struct efx_tc_ct_zone *ct_zone) { struct efx_tc_ct_entry *conn, *next; if (!refcount_dec_and_test(&ct_zone->ref)) return; /* still in use */ nf_flow_table_offload_del_cb(ct_zone->nf_ft, efx_tc_flow_block, ct_zone); rhashtable_remove_fast(&efx->tc->ct_zone_ht, &ct_zone->linkage, efx_tc_ct_zone_ht_params); mutex_lock(&ct_zone->mutex); list_for_each_entry(conn, &ct_zone->cts, list) efx_tc_ct_remove(efx, conn); synchronize_rcu(); /* need to use _safe because efx_tc_ct_remove_finish() frees conn */ list_for_each_entry_safe(conn, next, &ct_zone->cts, list) efx_tc_ct_remove_finish(efx, conn); mutex_unlock(&ct_zone->mutex); mutex_destroy(&ct_zone->mutex); netif_dbg(efx, drv, efx->net_dev, "Removed ct_zone for %u\n", ct_zone->zone); kfree(ct_zone); }
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1