cregit-Linux how code gets into the kernel

Release 4.14 net/sched/cls_flower.c

Directory: net/sched
/*
 * net/sched/cls_flower.c               Flower classifier
 *
 * Copyright (c) 2015 Jiri Pirko <jiri@resnulli.us>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 */

#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/rhashtable.h>
#include <linux/workqueue.h>

#include <linux/if_ether.h>
#include <linux/in6.h>
#include <linux/ip.h>
#include <linux/mpls.h>

#include <net/sch_generic.h>
#include <net/pkt_cls.h>
#include <net/ip.h>
#include <net/flow_dissector.h>

#include <net/dst.h>
#include <net/dst_metadata.h>


struct fl_flow_key {
	
int	indev_ifindex;
	
struct flow_dissector_key_control control;
	
struct flow_dissector_key_control enc_control;
	
struct flow_dissector_key_basic basic;
	
struct flow_dissector_key_eth_addrs eth;
	
struct flow_dissector_key_vlan vlan;
	union {
		
struct flow_dissector_key_ipv4_addrs ipv4;
		
struct flow_dissector_key_ipv6_addrs ipv6;
	};
	
struct flow_dissector_key_ports tp;
	
struct flow_dissector_key_icmp icmp;
	
struct flow_dissector_key_arp arp;
	
struct flow_dissector_key_keyid enc_key_id;
	union {
		
struct flow_dissector_key_ipv4_addrs enc_ipv4;
		
struct flow_dissector_key_ipv6_addrs enc_ipv6;
	};
	
struct flow_dissector_key_ports enc_tp;
	
struct flow_dissector_key_mpls mpls;
	
struct flow_dissector_key_tcp tcp;
	
struct flow_dissector_key_ip ip;
} __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */


struct fl_flow_mask_range {
	
unsigned short int start;
	
unsigned short int end;
};


struct fl_flow_mask {
	
struct fl_flow_key key;
	
struct fl_flow_mask_range range;
	
struct rcu_head	rcu;
};


struct cls_fl_head {
	
struct rhashtable ht;
	
struct fl_flow_mask mask;
	
struct flow_dissector dissector;
	
bool mask_assigned;
	
struct list_head filters;
	
struct rhashtable_params ht_params;
	union {
		
struct work_struct work;
		
struct rcu_head	rcu;
	};
	
struct idr handle_idr;
};


struct cls_fl_filter {
	
struct rhash_head ht_node;
	
struct fl_flow_key mkey;
	
struct tcf_exts exts;
	
struct tcf_result res;
	
struct fl_flow_key key;
	
struct list_head list;
	
u32 handle;
	
u32 flags;
	union {
		
struct work_struct work;
		
struct rcu_head	rcu;
	};
	
struct net_device *hw_dev;
};


static unsigned short int fl_mask_range(const struct fl_flow_mask *mask) { return mask->range.end - mask->range.start; }

Contributors

PersonTokensPropCommitsCommitProp
Jiri Pirko27100.00%1100.00%
Total27100.00%1100.00%


static void fl_mask_update_range(struct fl_flow_mask *mask) { const u8 *bytes = (const u8 *) &mask->key; size_t size = sizeof(mask->key); size_t i, first = 0, last = size - 1; for (i = 0; i < sizeof(mask->key); i++) { if (bytes[i]) { if (!first && i) first = i; last = i; } } mask->range.start = rounddown(first, sizeof(long)); mask->range.end = roundup(last + 1, sizeof(long)); }

Contributors

PersonTokensPropCommitsCommitProp
Jiri Pirko127100.00%1100.00%
Total127100.00%1100.00%


static void *fl_key_get_start(struct fl_flow_key *key, const struct fl_flow_mask *mask) { return (u8 *) key + mask->range.start; }

Contributors

PersonTokensPropCommitsCommitProp
Jiri Pirko31100.00%1100.00%
Total31100.00%1100.00%


static void fl_set_masked_key(struct fl_flow_key *mkey, struct fl_flow_key *key, struct fl_flow_mask *mask) { const long *lkey = fl_key_get_start(key, mask); const long *lmask = fl_key_get_start(&mask->key, mask); long *lmkey = fl_key_get_start(mkey, mask); int i; for (i = 0; i < fl_mask_range(mask); i += sizeof(long)) *lmkey++ = *lkey++ & *lmask++; }

Contributors

PersonTokensPropCommitsCommitProp
Jiri Pirko94100.00%1100.00%
Total94100.00%1100.00%


static void fl_clear_masked_range(struct fl_flow_key *key, struct fl_flow_mask *mask) { memset(fl_key_get_start(key, mask), 0, fl_mask_range(mask)); }

Contributors

PersonTokensPropCommitsCommitProp
Jiri Pirko33100.00%1100.00%
Total33100.00%1100.00%


static struct cls_fl_filter *fl_lookup(struct cls_fl_head *head, struct fl_flow_key *mkey) { return rhashtable_lookup_fast(&head->ht, fl_key_get_start(mkey, &head->mask), head->ht_params); }

Contributors

PersonTokensPropCommitsCommitProp
Paul Blakey41100.00%1100.00%
Total41100.00%1100.00%


static int fl_classify(struct sk_buff *skb, const struct tcf_proto *tp, struct tcf_result *res) { struct cls_fl_head *head = rcu_dereference_bh(tp->root); struct cls_fl_filter *f; struct fl_flow_key skb_key; struct fl_flow_key skb_mkey; struct ip_tunnel_info *info; if (!atomic_read(&head->ht.nelems)) return -1; fl_clear_masked_range(&skb_key, &head->mask); info = skb_tunnel_info(skb); if (info) { struct ip_tunnel_key *key = &info->key; switch (ip_tunnel_info_af(info)) { case AF_INET: skb_key.enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; skb_key.enc_ipv4.src = key->u.ipv4.src; skb_key.enc_ipv4.dst = key->u.ipv4.dst; break; case AF_INET6: skb_key.enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; skb_key.enc_ipv6.src = key->u.ipv6.src; skb_key.enc_ipv6.dst = key->u.ipv6.dst; break; } skb_key.enc_key_id.keyid = tunnel_id_to_key32(key->tun_id); skb_key.enc_tp.src = key->tp_src; skb_key.enc_tp.dst = key->tp_dst; } skb_key.indev_ifindex = skb->skb_iif; /* skb_flow_dissect() does not set n_proto in case an unknown protocol, * so do it rather here. */ skb_key.basic.n_proto = skb->protocol; skb_flow_dissect(skb, &head->dissector, &skb_key, 0); fl_set_masked_key(&skb_mkey, &skb_key, &head->mask); f = fl_lookup(head, &skb_mkey); if (f && !tc_skip_sw(f->flags)) { *res = f->res; return tcf_exts_exec(skb, &f->exts, res); } return -1; }

Contributors

PersonTokensPropCommitsCommitProp
Jiri Pirko14344.55%112.50%
Amir Vadai13943.30%337.50%
Hadar Hen Zion206.23%112.50%
Paul Blakey175.30%225.00%
Tom Herbert20.62%112.50%
Total321100.00%8100.00%


static int fl_init(struct tcf_proto *tp) { struct cls_fl_head *head; head = kzalloc(sizeof(*head), GFP_KERNEL); if (!head) return -ENOBUFS; INIT_LIST_HEAD_RCU(&head->filters); rcu_assign_pointer(tp->root, head); idr_init(&head->handle_idr); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Jiri Pirko5887.88%150.00%
Chris Mi812.12%150.00%
Total66100.00%2100.00%


static void __fl_destroy_filter(struct cls_fl_filter *f) { tcf_exts_destroy(&f->exts); tcf_exts_put_net(&f->exts); kfree(f); }

Contributors

PersonTokensPropCommitsCommitProp
Américo Wang32100.00%1100.00%
Total32100.00%1100.00%


static void fl_destroy_filter_work(struct work_struct *work) { struct cls_fl_filter *f = container_of(work, struct cls_fl_filter, work); rtnl_lock(); __fl_destroy_filter(f); rtnl_unlock(); }

Contributors

PersonTokensPropCommitsCommitProp
Américo Wang37100.00%2100.00%
Total37100.00%2100.00%


static void fl_destroy_filter(struct rcu_head *head) { struct cls_fl_filter *f = container_of(head, struct cls_fl_filter, rcu); INIT_WORK(&f->work, fl_destroy_filter_work); tcf_queue_work(&f->work); }

Contributors

PersonTokensPropCommitsCommitProp
Jiri Pirko3681.82%150.00%
Américo Wang818.18%150.00%
Total44100.00%2100.00%


static void fl_hw_destroy_filter(struct tcf_proto *tp, struct cls_fl_filter *f) { struct tc_cls_flower_offload cls_flower = {}; struct net_device *dev = f->hw_dev; if (!tc_can_offload(dev)) return; tc_cls_common_offload_init(&cls_flower.common, tp); cls_flower.command = TC_CLSFLOWER_DESTROY; cls_flower.cookie = (unsigned long) f; cls_flower.egress_dev = f->hw_dev != tp->q->dev_queue->dev; dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_CLSFLOWER, &cls_flower); }

Contributors

PersonTokensPropCommitsCommitProp
Amir Vadai4445.83%112.50%
Hadar Hen Zion1818.75%337.50%
Jiri Pirko1818.75%337.50%
Or Gerlitz1616.67%112.50%
Total96100.00%8100.00%


static int fl_hw_replace_filter(struct tcf_proto *tp, struct flow_dissector *dissector, struct fl_flow_key *mask, struct cls_fl_filter *f) { struct net_device *dev = tp->q->dev_queue->dev; struct tc_cls_flower_offload cls_flower = {}; int err; if (!tc_can_offload(dev)) { if (tcf_exts_get_dev(dev, &f->exts, &f->hw_dev) || (f->hw_dev && !tc_can_offload(f->hw_dev))) { f->hw_dev = dev; return tc_skip_sw(f->flags) ? -EINVAL : 0; } dev = f->hw_dev; cls_flower.egress_dev = true; } else { f->hw_dev = dev; } tc_cls_common_offload_init(&cls_flower.common, tp); cls_flower.command = TC_CLSFLOWER_REPLACE; cls_flower.cookie = (unsigned long) f; cls_flower.dissector = dissector; cls_flower.mask = mask; cls_flower.key = &f->mkey; cls_flower.exts = &f->exts; err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_CLSFLOWER, &cls_flower); if (!err) f->flags |= TCA_CLS_FLAGS_IN_HW; if (tc_skip_sw(f->flags)) return err; return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Amir Vadai11349.34%216.67%
Hadar Hen Zion8034.93%433.33%
Jiri Pirko2410.48%433.33%
Or Gerlitz114.80%18.33%
Paul Blakey10.44%18.33%
Total229100.00%12100.00%


static void fl_hw_update_stats(struct tcf_proto *tp, struct cls_fl_filter *f) { struct tc_cls_flower_offload cls_flower = {}; struct net_device *dev = f->hw_dev; if (!tc_can_offload(dev)) return; tc_cls_common_offload_init(&cls_flower.common, tp); cls_flower.command = TC_CLSFLOWER_STATS; cls_flower.cookie = (unsigned long) f; cls_flower.exts = &f->exts; cls_flower.egress_dev = f->hw_dev != tp->q->dev_queue->dev; dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_CLSFLOWER, &cls_flower); }

Contributors

PersonTokensPropCommitsCommitProp
Amir Vadai6158.10%112.50%
Jiri Pirko1918.10%450.00%
Or Gerlitz1615.24%112.50%
Hadar Hen Zion98.57%225.00%
Total105100.00%8100.00%


static void __fl_delete(struct tcf_proto *tp, struct cls_fl_filter *f) { struct cls_fl_head *head = rtnl_dereference(tp->root); idr_remove_ext(&head->handle_idr, f->handle); list_del_rcu(&f->list); if (!tc_skip_hw(f->flags)) fl_hw_destroy_filter(tp, f); tcf_unbind_filter(tp, &f->res); if (tcf_exts_get_net(&f->exts)) call_rcu(&f->rcu, fl_destroy_filter); else __fl_destroy_filter(f); }

Contributors

PersonTokensPropCommitsCommitProp
Roi Dayan5150.50%125.00%
Chris Mi2423.76%125.00%
Américo Wang1615.84%125.00%
Hadar Hen Zion109.90%125.00%
Total101100.00%4100.00%


static void fl_destroy_sleepable(struct work_struct *work) { struct cls_fl_head *head = container_of(work, struct cls_fl_head, work); if (head->mask_assigned) rhashtable_destroy(&head->ht); kfree(head); module_put(THIS_MODULE); }

Contributors

PersonTokensPropCommitsCommitProp
Daniel Borkmann50100.00%1100.00%
Total50100.00%1100.00%


static void fl_destroy_rcu(struct rcu_head *rcu) { struct cls_fl_head *head = container_of(rcu, struct cls_fl_head, rcu); INIT_WORK(&head->work, fl_destroy_sleepable); schedule_work(&head->work); }

Contributors

PersonTokensPropCommitsCommitProp
Daniel Borkmann44100.00%1100.00%
Total44100.00%1100.00%


static void fl_destroy(struct tcf_proto *tp) { struct cls_fl_head *head = rtnl_dereference(tp->root); struct cls_fl_filter *f, *next; list_for_each_entry_safe(f, next, &head->filters, list) __fl_delete(tp, f); idr_destroy(&head->handle_idr); __module_get(THIS_MODULE); call_rcu(&head->rcu, fl_destroy_rcu); }

Contributors

PersonTokensPropCommitsCommitProp
Jiri Pirko5070.42%114.29%
Chris Mi811.27%114.29%
Amir Vadai68.45%114.29%
Daniel Borkmann45.63%114.29%
Américo Wang11.41%114.29%
David S. Miller11.41%114.29%
Roi Dayan11.41%114.29%
Total71100.00%7100.00%


static void *fl_get(struct tcf_proto *tp, u32 handle) { struct cls_fl_head *head = rtnl_dereference(tp->root); return idr_find_ext(&head->handle_idr, handle); }

Contributors

PersonTokensPropCommitsCommitProp
Jiri Pirko2976.32%133.33%
Chris Mi718.42%133.33%
Américo Wang25.26%133.33%
Total38100.00%3100.00%

static const struct nla_policy fl_policy[TCA_FLOWER_MAX + 1] = { [TCA_FLOWER_UNSPEC] = { .type = NLA_UNSPEC }, [TCA_FLOWER_CLASSID] = { .type = NLA_U32 }, [TCA_FLOWER_INDEV] = { .type = NLA_STRING, .len = IFNAMSIZ }, [TCA_FLOWER_KEY_ETH_DST] = { .len = ETH_ALEN }, [TCA_FLOWER_KEY_ETH_DST_MASK] = { .len = ETH_ALEN }, [TCA_FLOWER_KEY_ETH_SRC] = { .len = ETH_ALEN }, [TCA_FLOWER_KEY_ETH_SRC_MASK] = { .len = ETH_ALEN }, [TCA_FLOWER_KEY_ETH_TYPE] = { .type = NLA_U16 }, [TCA_FLOWER_KEY_IP_PROTO] = { .type = NLA_U8 }, [TCA_FLOWER_KEY_IPV4_SRC] = { .type = NLA_U32 }, [TCA_FLOWER_KEY_IPV4_SRC_MASK] = { .type = NLA_U32 }, [TCA_FLOWER_KEY_IPV4_DST] = { .type = NLA_U32 }, [TCA_FLOWER_KEY_IPV4_DST_MASK] = { .type = NLA_U32 }, [TCA_FLOWER_KEY_IPV6_SRC] = { .len = sizeof(struct in6_addr) }, [TCA_FLOWER_KEY_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) }, [TCA_FLOWER_KEY_IPV6_DST] = { .len = sizeof(struct in6_addr) }, [TCA_FLOWER_KEY_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) }, [TCA_FLOWER_KEY_TCP_SRC] = { .type = NLA_U16 }, [TCA_FLOWER_KEY_TCP_DST] = { .type = NLA_U16 }, [TCA_FLOWER_KEY_UDP_SRC] = { .type = NLA_U16 }, [TCA_FLOWER_KEY_UDP_DST] = { .type = NLA_U16 }, [TCA_FLOWER_KEY_VLAN_ID] = { .type = NLA_U16 }, [TCA_FLOWER_KEY_VLAN_PRIO] = { .type = NLA_U8 }, [TCA_FLOWER_KEY_VLAN_ETH_TYPE] = { .type = NLA_U16 }, [TCA_FLOWER_KEY_ENC_KEY_ID] = { .type = NLA_U32 }, [TCA_FLOWER_KEY_ENC_IPV4_SRC] = { .type = NLA_U32 }, [TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK] = { .type = NLA_U32 }, [TCA_FLOWER_KEY_ENC_IPV4_DST] = { .type = NLA_U32 }, [TCA_FLOWER_KEY_ENC_IPV4_DST_MASK] = { .type = NLA_U32 }, [TCA_FLOWER_KEY_ENC_IPV6_SRC] = { .len = sizeof(struct in6_addr) }, [TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) }, [TCA_FLOWER_KEY_ENC_IPV6_DST] = { .len = sizeof(struct in6_addr) }, [TCA_FLOWER_KEY_ENC_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) }, [TCA_FLOWER_KEY_TCP_SRC_MASK] = { .type = NLA_U16 }, [TCA_FLOWER_KEY_TCP_DST_MASK] = { .type = NLA_U16 }, [TCA_FLOWER_KEY_UDP_SRC_MASK] = { .type = NLA_U16 }, [TCA_FLOWER_KEY_UDP_DST_MASK] = { .type = NLA_U16 }, [TCA_FLOWER_KEY_SCTP_SRC_MASK] = { .type = NLA_U16 }, [TCA_FLOWER_KEY_SCTP_DST_MASK] = { .type = NLA_U16 }, [TCA_FLOWER_KEY_SCTP_SRC] = { .type = NLA_U16 }, [TCA_FLOWER_KEY_SCTP_DST] = { .type = NLA_U16 }, [TCA_FLOWER_KEY_ENC_UDP_SRC_PORT] = { .type = NLA_U16 }, [TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK] = { .type = NLA_U16 }, [TCA_FLOWER_KEY_ENC_UDP_DST_PORT] = { .type = NLA_U16 }, [TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK] = { .type = NLA_U16 }, [TCA_FLOWER_KEY_FLAGS] = { .type = NLA_U32 }, [TCA_FLOWER_KEY_FLAGS_MASK] = { .type = NLA_U32 }, [TCA_FLOWER_KEY_ICMPV4_TYPE] = { .type = NLA_U8 }, [TCA_FLOWER_KEY_ICMPV4_TYPE_MASK] = { .type = NLA_U8 }, [TCA_FLOWER_KEY_ICMPV4_CODE] = { .type = NLA_U8 }, [TCA_FLOWER_KEY_ICMPV4_CODE_MASK] = { .type = NLA_U8 }, [TCA_FLOWER_KEY_ICMPV6_TYPE] = { .type = NLA_U8 }, [TCA_FLOWER_KEY_ICMPV6_TYPE_MASK] = { .type = NLA_U8 }, [TCA_FLOWER_KEY_ICMPV6_CODE] = { .type = NLA_U8 }, [TCA_FLOWER_KEY_ICMPV6_CODE_MASK] = { .type = NLA_U8 }, [TCA_FLOWER_KEY_ARP_SIP] = { .type = NLA_U32 }, [TCA_FLOWER_KEY_ARP_SIP_MASK] = { .type = NLA_U32 }, [TCA_FLOWER_KEY_ARP_TIP] = { .type = NLA_U32 }, [TCA_FLOWER_KEY_ARP_TIP_MASK] = { .type = NLA_U32 }, [TCA_FLOWER_KEY_ARP_OP] = { .type = NLA_U8 }, [TCA_FLOWER_KEY_ARP_OP_MASK] = { .type = NLA_U8 }, [TCA_FLOWER_KEY_ARP_SHA] = { .len = ETH_ALEN }, [TCA_FLOWER_KEY_ARP_SHA_MASK] = { .len = ETH_ALEN }, [TCA_FLOWER_KEY_ARP_THA] = { .len = ETH_ALEN }, [TCA_FLOWER_KEY_ARP_THA_MASK] = { .len = ETH_ALEN }, [TCA_FLOWER_KEY_MPLS_TTL] = { .type = NLA_U8 }, [TCA_FLOWER_KEY_MPLS_BOS] = { .type = NLA_U8 }, [TCA_FLOWER_KEY_MPLS_TC] = { .type = NLA_U8 }, [TCA_FLOWER_KEY_MPLS_LABEL] = { .type = NLA_U32 }, [TCA_FLOWER_KEY_TCP_FLAGS] = { .type = NLA_U16 }, [TCA_FLOWER_KEY_TCP_FLAGS_MASK] = { .type = NLA_U16 }, [TCA_FLOWER_KEY_IP_TOS] = { .type = NLA_U8 }, [TCA_FLOWER_KEY_IP_TOS_MASK] = { .type = NLA_U8 }, [TCA_FLOWER_KEY_IP_TTL] = { .type = NLA_U8 }, [TCA_FLOWER_KEY_IP_TTL_MASK] = { .type = NLA_U8 }, };
static void fl_set_key_val(struct nlattr **tb, void *val, int val_type, void *mask, int mask_type, int len) { if (!tb[val_type]) return; memcpy(val, nla_data(tb[val_type]), len); if (mask_type == TCA_FLOWER_UNSPEC || !tb[mask_type]) memset(mask, 0xff, len); else memcpy(mask, nla_data(tb[mask_type]), len); }

Contributors

PersonTokensPropCommitsCommitProp
Jiri Pirko90100.00%1100.00%
Total90100.00%1100.00%


static int fl_set_key_mpls(struct nlattr **tb, struct flow_dissector_key_mpls *key_val, struct flow_dissector_key_mpls *key_mask) { if (tb[TCA_FLOWER_KEY_MPLS_TTL]) { key_val->mpls_ttl = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_TTL]); key_mask->mpls_ttl = MPLS_TTL_MASK; } if (tb[TCA_FLOWER_KEY_MPLS_BOS]) { u8 bos = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_BOS]); if (bos & ~MPLS_BOS_MASK) return -EINVAL; key_val->mpls_bos = bos; key_mask->mpls_bos = MPLS_BOS_MASK; } if (tb[TCA_FLOWER_KEY_MPLS_TC]) { u8 tc = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_TC]); if (tc & ~MPLS_TC_MASK) return -EINVAL; key_val->mpls_tc = tc; key_mask->mpls_tc = MPLS_TC_MASK; } if (tb[TCA_FLOWER_KEY_MPLS_LABEL]) { u32 label = nla_get_u32(tb[TCA_FLOWER_KEY_MPLS_LABEL]); if (label & ~MPLS_LABEL_MASK) return -EINVAL; key_val->mpls_label = label; key_mask->mpls_label = MPLS_LABEL_MASK; } return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Benjamin LaHaise181100.00%2100.00%
Total181100.00%2100.00%


static void fl_set_key_vlan(struct nlattr **tb, struct flow_dissector_key_vlan *key_val, struct flow_dissector_key_vlan *key_mask)