cregit-Linux how code gets into the kernel

Release 4.11 net/sched/cls_flow.c

Directory: net/sched
/*
 * net/sched/cls_flow.c         Generic flow classifier
 *
 * Copyright (c) 2007, 2008 Patrick McHardy <kaber@trash.net>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 */

#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/list.h>
#include <linux/jhash.h>
#include <linux/random.h>
#include <linux/pkt_cls.h>
#include <linux/skbuff.h>
#include <linux/in.h>
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/if_vlan.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <net/inet_sock.h>

#include <net/pkt_cls.h>
#include <net/ip.h>
#include <net/route.h>
#include <net/flow_dissector.h>

#if IS_ENABLED(CONFIG_NF_CONNTRACK)
#include <net/netfilter/nf_conntrack.h>
#endif


struct flow_head {
	
struct list_head	filters;
	
struct rcu_head		rcu;
};


struct flow_filter {
	
struct list_head	list;
	
struct tcf_exts		exts;
	
struct tcf_ematch_tree	ematches;
	
struct tcf_proto	*tp;
	
struct timer_list	perturb_timer;
	
u32			perturb_period;
	
u32			handle;

	
u32			nkeys;
	
u32			keymask;
	
u32			mode;
	
u32			mask;
	
u32			xor;
	
u32			rshift;
	
u32			addend;
	
u32			divisor;
	
u32			baseclass;
	
u32			hashrnd;
	
struct rcu_head		rcu;
};


static inline u32 addr_fold(void *addr) { unsigned long a = (unsigned long)addr; return (a & 0xFFFFFFFF) ^ (BITS_PER_LONG > 32 ? a >> 32 : 0); }

Contributors

PersonTokensPropCommitsCommitProp
Patrick McHardy40100.00%1100.00%
Total40100.00%1100.00%


static u32 flow_get_src(const struct sk_buff *skb, const struct flow_keys *flow) { __be32 src = flow_get_u32_src(flow); if (src) return ntohl(src); return addr_fold(skb->sk); }

Contributors

PersonTokensPropCommitsCommitProp
Patrick McHardy1840.91%125.00%
Eric Dumazet1840.91%250.00%
Tom Herbert818.18%125.00%
Total44100.00%4100.00%


static u32 flow_get_dst(const struct sk_buff *skb, const struct flow_keys *flow) { __be32 dst = flow_get_u32_dst(flow); if (dst) return ntohl(dst); return addr_fold(skb_dst(skb)) ^ (__force u16) tc_skb_protocol(skb); }

Contributors

PersonTokensPropCommitsCommitProp
Patrick McHardy2240.74%116.67%
Eric Dumazet2138.89%350.00%
Tom Herbert814.81%116.67%
Jiri Pirko35.56%116.67%
Total54100.00%6100.00%


static u32 flow_get_proto(const struct sk_buff *skb, const struct flow_keys *flow) { return flow->basic.ip_proto; }

Contributors

PersonTokensPropCommitsCommitProp
Eric Dumazet1248.00%250.00%
Patrick McHardy1144.00%125.00%
Jiri Pirko28.00%125.00%
Total25100.00%4100.00%


static u32 flow_get_proto_src(const struct sk_buff *skb, const struct flow_keys *flow) { if (flow->ports.ports) return ntohs(flow->ports.src); return addr_fold(skb->sk); }

Contributors

PersonTokensPropCommitsCommitProp
Eric Dumazet3375.00%228.57%
Jiri Pirko511.36%228.57%
Changli Gao49.09%228.57%
Patrick McHardy24.55%114.29%
Total44100.00%7100.00%


static u32 flow_get_proto_dst(const struct sk_buff *skb, const struct flow_keys *flow) { if (flow->ports.ports) return ntohs(flow->ports.dst); return addr_fold(skb_dst(skb)) ^ (__force u16) tc_skb_protocol(skb); }

Contributors

PersonTokensPropCommitsCommitProp
Eric Dumazet3157.41%337.50%
Patrick McHardy1324.07%112.50%
Jiri Pirko814.81%337.50%
Changli Gao23.70%112.50%
Total54100.00%8100.00%


static u32 flow_get_iif(const struct sk_buff *skb) { return skb->skb_iif; }

Contributors

PersonTokensPropCommitsCommitProp
Patrick McHardy1694.12%150.00%
Eric Dumazet15.88%150.00%
Total17100.00%2100.00%


static u32 flow_get_priority(const struct sk_buff *skb) { return skb->priority; }

Contributors

PersonTokensPropCommitsCommitProp
Patrick McHardy17100.00%1100.00%
Total17100.00%1100.00%


static u32 flow_get_mark(const struct sk_buff *skb) { return skb->mark; }

Contributors

PersonTokensPropCommitsCommitProp
Patrick McHardy17100.00%1100.00%
Total17100.00%1100.00%


static u32 flow_get_nfct(const struct sk_buff *skb) { #if IS_ENABLED(CONFIG_NF_CONNTRACK) return addr_fold(skb_nfct(skb)); #else return 0; #endif }

Contributors

PersonTokensPropCommitsCommitProp
Patrick McHardy3088.24%133.33%
Florian Westphal38.82%133.33%
Javier Martinez Canillas12.94%133.33%
Total34100.00%3100.00%

#if IS_ENABLED(CONFIG_NF_CONNTRACK) #define CTTUPLE(skb, member) \ ({ \ enum ip_conntrack_info ctinfo; \ const struct nf_conn *ct = nf_ct_get(skb, &ctinfo); \ if (ct == NULL) \ goto fallback; \ ct->tuplehash[CTINFO2DIR(ctinfo)].tuple.member; \ }) #else #define CTTUPLE(skb, member) \ ({ \ goto fallback; \ 0; \ }) #endif
static u32 flow_get_nfct_src(const struct sk_buff *skb, const struct flow_keys *flow) { switch (tc_skb_protocol(skb)) { case htons(ETH_P_IP): return ntohl(CTTUPLE(skb, src.u3.ip)); case htons(ETH_P_IPV6): return ntohl(CTTUPLE(skb, src.u3.ip6[3])); } fallback: return flow_get_src(skb, flow); }

Contributors

PersonTokensPropCommitsCommitProp
Patrick McHardy6882.93%120.00%
Eric Dumazet910.98%240.00%
Jiri Pirko33.66%120.00%
Arnaldo Carvalho de Melo22.44%120.00%
Total82100.00%5100.00%


static u32 flow_get_nfct_dst(const struct sk_buff *skb, const struct flow_keys *flow) { switch (tc_skb_protocol(skb)) { case htons(ETH_P_IP): return ntohl(CTTUPLE(skb, dst.u3.ip)); case htons(ETH_P_IPV6): return ntohl(CTTUPLE(skb, dst.u3.ip6[3])); } fallback: return flow_get_dst(skb, flow); }

Contributors

PersonTokensPropCommitsCommitProp
Patrick McHardy6882.93%120.00%
Eric Dumazet910.98%240.00%
Jiri Pirko33.66%120.00%
Arnaldo Carvalho de Melo22.44%120.00%
Total82100.00%5100.00%


static u32 flow_get_nfct_proto_src(const struct sk_buff *skb, const struct flow_keys *flow) { return ntohs(CTTUPLE(skb, src.u.all)); fallback: return flow_get_proto_src(skb, flow); }

Contributors

PersonTokensPropCommitsCommitProp
Patrick McHardy3479.07%133.33%
Eric Dumazet920.93%266.67%
Total43100.00%3100.00%


static u32 flow_get_nfct_proto_dst(const struct sk_buff *skb, const struct flow_keys *flow) { return ntohs(CTTUPLE(skb, dst.u.all)); fallback: return flow_get_proto_dst(skb, flow); }

Contributors

PersonTokensPropCommitsCommitProp
Patrick McHardy3479.07%133.33%
Eric Dumazet920.93%266.67%
Total43100.00%3100.00%


static u32 flow_get_rtclassid(const struct sk_buff *skb) { #ifdef CONFIG_IP_ROUTE_CLASSID if (skb_dst(skb)) return skb_dst(skb)->tclassid; #endif return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Patrick McHardy2982.86%266.67%
Eric Dumazet617.14%133.33%
Total35100.00%3100.00%


static u32 flow_get_skuid(const struct sk_buff *skb) { struct sock *sk = skb_to_full_sk(skb); if (sk && sk->sk_socket && sk->sk_socket->file) { kuid_t skuid = sk->sk_socket->file->f_cred->fsuid; return from_kuid(&init_user_ns, skuid); } return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Patrick McHardy3657.14%125.00%
Eric W. Biedermann1422.22%125.00%
Eric Dumazet1015.87%125.00%
David Howells34.76%125.00%
Total63100.00%4100.00%


static u32 flow_get_skgid(const struct sk_buff *skb) { struct sock *sk = skb_to_full_sk(skb); if (sk && sk->sk_socket && sk->sk_socket->file) { kgid_t skgid = sk->sk_socket->file->f_cred->fsgid; return from_kgid(&init_user_ns, skgid); } return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Patrick McHardy3657.14%125.00%
Eric W. Biedermann1422.22%125.00%
Eric Dumazet1015.87%125.00%
David Howells34.76%125.00%
Total63100.00%4100.00%


static u32 flow_get_vlan_tag(const struct sk_buff *skb) { u16 uninitialized_var(tag); if (vlan_get_tag(skb, &tag) < 0) return 0; return tag & VLAN_VID_MASK; }

Contributors

PersonTokensPropCommitsCommitProp
Patrick McHardy38100.00%1100.00%
Total38100.00%1100.00%


static u32 flow_get_rxhash(struct sk_buff *skb) { return skb_get_hash(skb); }

Contributors

PersonTokensPropCommitsCommitProp
Changli Gao1694.12%150.00%
Tom Herbert15.88%150.00%
Total17100.00%2100.00%


static u32 flow_key_get(struct sk_buff *skb, int key, struct flow_keys *flow) { switch (key) { case FLOW_KEY_SRC: return flow_get_src(skb, flow); case FLOW_KEY_DST: return flow_get_dst(skb, flow); case FLOW_KEY_PROTO: return flow_get_proto(skb, flow); case FLOW_KEY_PROTO_SRC: return flow_get_proto_src(skb, flow); case FLOW_KEY_PROTO_DST: return flow_get_proto_dst(skb, flow); case FLOW_KEY_IIF: return flow_get_iif(skb); case FLOW_KEY_PRIORITY: return flow_get_priority(skb); case FLOW_KEY_MARK: return flow_get_mark(skb); case FLOW_KEY_NFCT: return flow_get_nfct(skb); case FLOW_KEY_NFCT_SRC: return flow_get_nfct_src(skb, flow); case FLOW_KEY_NFCT_DST: return flow_get_nfct_dst(skb, flow); case FLOW_KEY_NFCT_PROTO_SRC: return flow_get_nfct_proto_src(skb, flow); case FLOW_KEY_NFCT_PROTO_DST: return flow_get_nfct_proto_dst(skb, flow); case FLOW_KEY_RTCLASSID: return flow_get_rtclassid(skb); case FLOW_KEY_SKUID: return flow_get_skuid(skb); case FLOW_KEY_SKGID: return flow_get_skgid(skb); case FLOW_KEY_VLAN_TAG: return flow_get_vlan_tag(skb); case FLOW_KEY_RXHASH: return flow_get_rxhash(skb); default: WARN_ON(1); return 0; } }

Contributors

PersonTokensPropCommitsCommitProp
Patrick McHardy18285.05%240.00%
Eric Dumazet2310.75%240.00%
Changli Gao94.21%120.00%
Total214100.00%5100.00%

#define FLOW_KEYS_NEEDED ((1 << FLOW_KEY_SRC) | \ (1 << FLOW_KEY_DST) | \ (1 << FLOW_KEY_PROTO) | \ (1 << FLOW_KEY_PROTO_SRC) | \ (1 << FLOW_KEY_PROTO_DST) | \ (1 << FLOW_KEY_NFCT_SRC) | \ (1 << FLOW_KEY_NFCT_DST) | \ (1 << FLOW_KEY_NFCT_PROTO_SRC) | \ (1 << FLOW_KEY_NFCT_PROTO_DST))
static int flow_classify(struct sk_buff *skb, const struct tcf_proto *tp, struct tcf_result *res) { struct flow_head *head = rcu_dereference_bh(tp->root); struct flow_filter *f; u32 keymask; u32 classid; unsigned int n, key; int r; list_for_each_entry_rcu(f, &head->filters, list) { u32 keys[FLOW_KEY_MAX + 1]; struct flow_keys flow_keys; if (!tcf_em_tree_match(skb, &f->ematches, NULL)) continue; keymask = f->keymask; if (keymask & FLOW_KEYS_NEEDED) skb_flow_dissect_flow_keys(skb, &flow_keys, 0); for (n = 0; n < f->nkeys; n++) { key = ffs(keymask) - 1; keymask &= ~(1 << key); keys[n] = flow_key_get(skb, key, &flow_keys); } if (f->mode == FLOW_MODE_HASH) classid = jhash2(keys, f->nkeys, f->hashrnd); else { classid = keys[0]; classid = (classid & f->mask) ^ f->xor; classid = (classid >> f->rshift) + f->addend; } if (f->divisor) classid %= f->divisor; res->class = 0; res->classid = TC_H_MAKE(f->baseclass, f->baseclass + classid); r = tcf_exts_exec(skb, &f->exts, res); if (r < 0) continue; return r; } return -1; }

Contributors

PersonTokensPropCommitsCommitProp
Patrick McHardy25789.24%225.00%
Eric Dumazet248.33%337.50%
John Fastabend41.39%112.50%
Tom Herbert20.69%112.50%
Jiri Pirko10.35%112.50%
Total288100.00%8100.00%


static void flow_perturbation(unsigned long arg) { struct flow_filter *f = (struct flow_filter *)arg; get_random_bytes(&f->hashrnd, 4); if (f->perturb_period) mod_timer(&f->perturb_timer, jiffies + f->perturb_period); }

Contributors

PersonTokensPropCommitsCommitProp
Patrick McHardy52100.00%1100.00%
Total52100.00%1100.00%

static const struct nla_policy flow_policy[TCA_FLOW_MAX + 1] = { [TCA_FLOW_KEYS] = { .type = NLA_U32 }, [TCA_FLOW_MODE] = { .type = NLA_U32 }, [TCA_FLOW_BASECLASS] = { .type = NLA_U32 }, [TCA_FLOW_RSHIFT] = { .type = NLA_U32 }, [TCA_FLOW_ADDEND] = { .type = NLA_U32 }, [TCA_FLOW_MASK] = { .type = NLA_U32 }, [TCA_FLOW_XOR] = { .type = NLA_U32 }, [TCA_FLOW_DIVISOR] = { .type = NLA_U32 }, [TCA_FLOW_ACT] = { .type = NLA_NESTED }, [TCA_FLOW_POLICE] = { .type = NLA_NESTED }, [TCA_FLOW_EMATCHES] = { .type = NLA_NESTED }, [TCA_FLOW_PERTURB] = { .type = NLA_U32 }, };
static void flow_destroy_filter(struct rcu_head *head) { struct flow_filter *f = container_of(head, struct flow_filter, rcu); del_timer_sync(&f->perturb_timer); tcf_exts_destroy(&f->exts); tcf_em_tree_destroy(&f->ematches); kfree(f); }

Contributors

PersonTokensPropCommitsCommitProp
John Fastabend55100.00%1100.00%
Total55100.00%1100.00%


static int flow_change(struct net *net, struct sk_buff *in_skb, struct tcf_proto *tp, unsigned long base, u32 handle, struct nlattr **tca, unsigned long *arg, bool ovr) { struct flow_head *head = rtnl_dereference(tp->root); struct flow_filter *fold, *fnew; struct nlattr *opt = tca[TCA_OPTIONS]; struct nlattr *tb[TCA_FLOW_MAX + 1]; struct tcf_exts e; struct tcf_ematch_tree t; unsigned int nkeys = 0; unsigned int perturb_period = 0; u32 baseclass = 0; u32 keymask = 0; u32 mode; int err; if (opt == NULL) return -EINVAL; err = nla_parse_nested(tb, TCA_FLOW_MAX, opt, flow_policy); if (err < 0) return err; if (tb[TCA_FLOW_BASECLASS]) { baseclass = nla_get_u32(tb[TCA_FLOW_BASECLASS]); if (TC_H_MIN(baseclass) == 0) return -EINVAL; } if (tb[TCA_FLOW_KEYS]) { keymask = nla_get_u32(tb[TCA_FLOW_KEYS]); nkeys = hweight32(keymask); if (nkeys == 0) return -EINVAL; if (fls(keymask) - 1 > FLOW_KEY_MAX) return -EOPNOTSUPP; if ((keymask & (FLOW_KEY_SKUID|FLOW_KEY_SKGID)) && sk_user_ns(NETLINK_CB(in_skb).sk) != &init_user_ns) return -EOPNOTSUPP; } err = tcf_exts_init(&e, TCA_FLOW_ACT, TCA_FLOW_POLICE); if (err < 0) goto err1; err = tcf_exts_validate(net, tp, tb, tca[TCA_RATE], &e, ovr); if (err < 0) goto err1; err = tcf_em_tree_validate(tp, tb[TCA_FLOW_EMATCHES], &t); if (err < 0) goto err1; err = -ENOBUFS; fnew = kzalloc(sizeof(*fnew), GFP_KERNEL); if (!fnew) goto err2; err = tcf_exts_init(&fnew->exts, TCA_FLOW_ACT, TCA_FLOW_POLICE); if (err < 0) goto err3; fold = (struct flow_filter *)*arg; if (fold) { err = -EINVAL; if (fold->handle != handle && handle) goto err3; /* Copy fold into fnew */ fnew->tp = fold->tp; fnew->handle = fold->handle; fnew->nkeys = fold->nkeys; fnew->keymask = fold->keymask; fnew->mode = fold->mode; fnew->mask = fold->mask; fnew->xor = fold->xor; fnew->rshift = fold->rshift; fnew->addend = fold->addend; fnew->divisor = fold->divisor; fnew->baseclass = fold->baseclass; fnew->hashrnd = fold->hashrnd; mode = fold->mode; if (tb[TCA_FLOW_MODE]) mode = nla_get_u32(tb[TCA_FLOW_MODE]); if (mode != FLOW_MODE_HASH && nkeys > 1) goto err3; if (mode == FLOW_MODE_HASH) perturb_period = fold->perturb_period; if (tb[TCA_FLOW_PERTURB]) { if (mode != FLOW_MODE_HASH) goto err3; perturb_period = nla_get_u32(tb[TCA_FLOW_PERTURB]) * HZ; } } else { err = -EINVAL; if (!handle) goto err3; if (!tb[TCA_FLOW_KEYS]) goto err3; mode = FLOW_MODE_MAP; if (tb[TCA_FLOW_MODE]) mode = nla_get_u32(tb[TCA_FLOW_MODE]); if (mode != FLOW_MODE_HASH && nkeys > 1) goto err3; if (tb[TCA_FLOW_PERTURB]) { if (mode != FLOW_MODE_HASH) goto err3; perturb_period = nla_get_u32(tb[TCA_FLOW_PERTURB]) * HZ; } if (TC_H_MAJ(baseclass) == 0) baseclass = TC_H_MAKE(tp->q->handle, baseclass); if (TC_H_MIN(baseclass) == 0) baseclass = TC_H_MAKE(baseclass, 1); fnew->handle = handle; fnew->mask = ~0U; fnew->tp = tp; get_random_bytes(&fnew->hashrnd, 4); } fnew->perturb_timer.function = flow_perturbation; fnew->perturb_timer.data = (unsigned long)fnew; init_timer_deferrable(&fnew->perturb_timer); tcf_exts_change(tp, &fnew->exts, &e); tcf_em_tree_change(tp, &fnew->ematches, &t); netif_keep_dst(qdisc_dev(tp->q)); if (tb[TCA_FLOW_KEYS]) { fnew->keymask = keymask; fnew->nkeys = nkeys; } fnew->mode = mode; if (tb[TCA_FLOW_MASK]) fnew->mask = nla_get_u32(tb[TCA_FLOW_MASK]); if (tb[TCA_FLOW_XOR]) fnew->xor = nla_get_u32(tb[TCA_FLOW_XOR]); if (tb[TCA_FLOW_RSHIFT]) fnew->rshift = nla_get_u32(tb[TCA_FLOW_RSHIFT]); if (tb[TCA_FLOW_ADDEND]) fnew->addend = nla_get_u32(tb[TCA_FLOW_ADDEND]); if (tb[TCA_FLOW_DIVISOR]) fnew->divisor = nla_get_u32(tb[TCA_FLOW_DIVISOR]); if (baseclass) fnew->baseclass = baseclass; fnew->perturb_period = perturb_period; if (perturb_period) mod_timer(&fnew->perturb_timer, jiffies + perturb_period); if (*arg == 0) list_add_tail_rcu(&fnew->list, &head->filters); else list_replace_rcu(&fold->list, &fnew->list); *arg = (unsigned long)fnew; if (fold) call_rcu(&fold->rcu, flow_destroy_filter); return 0; err3: tcf_exts_destroy(&fnew->exts); err2: tcf_em_tree_destroy(&t); kfree(fnew); err1: tcf_exts_destroy(&e); return err; }

Contributors

PersonTokensPropCommitsCommitProp
Patrick McHardy74469.73%428.57%
John Fastabend19618.37%17.14%
Américo Wang585.44%321.43%
Eric W. Biedermann333.09%214.29%
Daniel Borkmann141.31%17.14%
Eric Dumazet100.94%17.14%
Benjamin LaHaise70.66%17.14%
Cong Wang50.47%17.14%
Total1067100.00%14100.00%


static int flow_delete(struct tcf_proto *tp, unsigned long arg) { struct flow_filter *f = (struct flow_filter *)arg; list_del_rcu(&f->list); call_rcu(&f->rcu, flow_destroy_filter); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Patrick McHardy4185.42%150.00%
John Fastabend714.58%150.00%
Total48100.00%2100.00%


static int flow_init(struct tcf_proto *tp) { struct flow_head *head; head = kzalloc(sizeof(*head), GFP_KERNEL); if (head == NULL) return -ENOBUFS; INIT_LIST_HEAD(&head->filters); rcu_assign_pointer(tp->root, head); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Patrick McHardy5593.22%150.00%
John Fastabend46.78%150.00%
Total59100.00%2100.00%


static bool flow_destroy(struct tcf_proto *tp, bool force) { struct flow_head *head = rtnl_dereference(tp->root); struct flow_filter *f, *next; if (!force && !list_empty(&head->filters)) return false; list_for_each_entry_safe(f, next, &head->filters, list) { list_del_rcu(&f->list); call_rcu(&f->rcu, flow_destroy_filter); } kfree_rcu(head, rcu); return true; }

Contributors

PersonTokensPropCommitsCommitProp
Patrick McHardy5054.95%133.33%
Cong Wang2426.37%133.33%
John Fastabend1718.68%133.33%
Total91100.00%3100.00%


static unsigned long flow_get(struct tcf_proto *tp, u32 handle) { struct flow_head *head = rtnl_dereference(tp->root); struct flow_filter *f; list_for_each_entry(f, &head->filters, list) if (f->handle == handle) return (unsigned long)f; return 0