Release 4.14 net/netfilter/nft_set_hash.c

/*
 * Copyright (c) 2008-2014 Patrick McHardy <kaber@trash.net>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * Development of this code funded by Astaro AG (http://www.astaro.com/)
 */

#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/list.h>
#include <linux/log2.h>
#include <linux/jhash.h>
#include <linux/netlink.h>
#include <linux/workqueue.h>
#include <linux/rhashtable.h>
#include <linux/netfilter.h>
#include <linux/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables.h>

/* We target a hash table size of 4, element hint is 75% of final size */

#define NFT_RHASH_ELEMENT_HINT 3


struct nft_rhash {
	
struct rhashtable		ht;
	
struct delayed_work		gc_work;
};


struct nft_rhash_elem {
	
struct rhash_head		node;
	
struct nft_set_ext		ext;
};


struct nft_rhash_cmp_arg {
	
const struct nft_set		*set;
	
const u32			*key;
	
u8				genmask;
};



static inline u32 nft_rhash_key(const void *data, u32 len, u32 seed)
{
	const struct nft_rhash_cmp_arg *arg = data;

	return jhash(arg->key, len, seed);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Patrick McHardy
36
94.74%
1
50.00%
Pablo Neira Ayuso
2
5.26%
1
50.00%
Total
38
100.00%
2
100.00%



static inline u32 nft_rhash_obj(const void *data, u32 len, u32 seed)
{
	const struct nft_rhash_elem *he = data;

	return jhash(nft_set_ext_key(&he->ext), len, seed);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Patrick McHardy
40
95.24%
2
66.67%
Pablo Neira Ayuso
2
4.76%
1
33.33%
Total
42
100.00%
3
100.00%



static inline int nft_rhash_cmp(struct rhashtable_compare_arg *arg,
				const void *ptr)
{
	const struct nft_rhash_cmp_arg *x = arg->key;
	const struct nft_rhash_elem *he = ptr;

	if (memcmp(nft_set_ext_key(&he->ext), x->key, x->set->klen))
		return 1;
	if (nft_set_elem_expired(&he->ext))
		return 1;
	if (!nft_set_elem_active(&he->ext, x->genmask))
		return 1;
	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Patrick McHardy
92
96.84%
5
83.33%
Pablo Neira Ayuso
3
3.16%
1
16.67%
Total
95
100.00%
6
100.00%


static const struct rhashtable_params nft_rhash_params = {
	.head_offset		= offsetof(struct nft_rhash_elem, node),
	.hashfn			= nft_rhash_key,
	.obj_hashfn		= nft_rhash_obj,
	.obj_cmpfn		= nft_rhash_cmp,
	.automatic_shrinking	= true,
};



static bool nft_rhash_lookup(const struct net *net, const struct nft_set *set,
			     const u32 *key, const struct nft_set_ext **ext)
{
	struct nft_rhash *priv = nft_set_priv(set);
	const struct nft_rhash_elem *he;
	struct nft_rhash_cmp_arg arg = {
		.genmask = nft_genmask_cur(net),
		.set	 = set,
		.key	 = key,
        };

	he = rhashtable_lookup_fast(&priv->ht, &arg, nft_rhash_params);
	if (he != NULL)
		*ext = &he->ext;

	return !!he;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Patrick McHardy
71
68.27%
7
63.64%
Thomas Graf
19
18.27%
1
9.09%
Pablo Neira Ayuso
12
11.54%
2
18.18%
Herbert Xu
2
1.92%
1
9.09%
Total
104
100.00%
11
100.00%



static bool nft_rhash_update(struct nft_set *set, const u32 *key,
			     void *(*new)(struct nft_set *,
					  const struct nft_expr *,
					  struct nft_regs *regs),
			     const struct nft_expr *expr,
			     struct nft_regs *regs,
			     const struct nft_set_ext **ext)
{
	struct nft_rhash *priv = nft_set_priv(set);
	struct nft_rhash_elem *he, *prev;
	struct nft_rhash_cmp_arg arg = {
		.genmask = NFT_GENMASK_ANY,
		.set	 = set,
		.key	 = key,
        };

	he = rhashtable_lookup_fast(&priv->ht, &arg, nft_rhash_params);
	if (he != NULL)
		goto out;

	he = new(set, expr, regs);
	if (he == NULL)
		goto err1;

	prev = rhashtable_lookup_get_insert_key(&priv->ht, &arg, &he->node,
						nft_rhash_params);
	if (IS_ERR(prev))
		goto err2;

	/* Another cpu may race to insert the element with the same key */
	if (prev) {
		nft_set_elem_destroy(set, he, true);
		he = prev;
	}

out:
	*ext = &he->ext;
	return true;

err2:
	nft_set_elem_destroy(set, he, true);
err1:
	return false;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Patrick McHardy
177
81.19%
3
50.00%
Liping Zhang
35
16.06%
2
33.33%
Pablo Neira Ayuso
6
2.75%
1
16.67%
Total
218
100.00%
6
100.00%



static int nft_rhash_insert(const struct net *net, const struct nft_set *set,
			    const struct nft_set_elem *elem,
			    struct nft_set_ext **ext)
{
	struct nft_rhash *priv = nft_set_priv(set);
	struct nft_rhash_elem *he = elem->priv;
	struct nft_rhash_cmp_arg arg = {
		.genmask = nft_genmask_next(net),
		.set	 = set,
		.key	 = elem->key.val.data,
        };
	struct nft_rhash_elem *prev;

	prev = rhashtable_lookup_get_insert_key(&priv->ht, &arg, &he->node,
						nft_rhash_params);
	if (IS_ERR(prev))
		return PTR_ERR(prev);
	if (prev) {
		*ext = &prev->ext;
		return -EEXIST;
	}
	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Patrick McHardy
73
52.90%
7
58.33%
Pablo Neira Ayuso
60
43.48%
3
25.00%
Thomas Graf
3
2.17%
1
8.33%
Herbert Xu
2
1.45%
1
8.33%
Total
138
100.00%
12
100.00%



static void nft_rhash_activate(const struct net *net, const struct nft_set *set,
			       const struct nft_set_elem *elem)
{
	struct nft_rhash_elem *he = elem->priv;

	nft_set_elem_change_active(net, set, &he->ext);
	nft_set_elem_clear_busy(&he->ext);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Patrick McHardy
42
79.25%
5
62.50%
Pablo Neira Ayuso
10
18.87%
2
25.00%
Thomas Graf
1
1.89%
1
12.50%
Total
53
100.00%
8
100.00%



static bool nft_rhash_flush(const struct net *net,
			    const struct nft_set *set, void *priv)
{
	struct nft_rhash_elem *he = priv;

	if (!nft_set_elem_mark_busy(&he->ext) ||
	    !nft_is_active(net, &he->ext)) {
		nft_set_elem_change_active(net, set, &he->ext);
		return true;
	}
	return false;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Pablo Neira Ayuso
71
100.00%
2
100.00%
Total
71
100.00%
2
100.00%



static void *nft_rhash_deactivate(const struct net *net,
				  const struct nft_set *set,
				  const struct nft_set_elem *elem)
{
	struct nft_rhash *priv = nft_set_priv(set);
	struct nft_rhash_elem *he;
	struct nft_rhash_cmp_arg arg = {
		.genmask = nft_genmask_next(net),
		.set	 = set,
		.key	 = elem->key.val.data,
        };

	rcu_read_lock();
	he = rhashtable_lookup_fast(&priv->ht, &arg, nft_rhash_params);
	if (he != NULL &&
	    !nft_rhash_flush(net, set, he))
		he = NULL;

	rcu_read_unlock();

	return he;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Patrick McHardy
78
68.42%
9
64.29%
Herbert Xu
19
16.67%
1
7.14%
Pablo Neira Ayuso
17
14.91%
4
28.57%
Total
114
100.00%
14
100.00%



static void nft_rhash_remove(const struct net *net,
			     const struct nft_set *set,
			     const struct nft_set_elem *elem)
{
	struct nft_rhash *priv = nft_set_priv(set);
	struct nft_rhash_elem *he = elem->priv;

	rhashtable_remove_fast(&priv->ht, &he->node, nft_rhash_params);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Patrick McHardy
46
79.31%
4
57.14%
Pablo Neira Ayuso
10
17.24%
2
28.57%
Thomas Graf
2
3.45%
1
14.29%
Total
58
100.00%
7
100.00%



static void nft_rhash_walk(const struct nft_ctx *ctx, struct nft_set *set,
			   struct nft_set_iter *iter)
{
	struct nft_rhash *priv = nft_set_priv(set);
	struct nft_rhash_elem *he;
	struct rhashtable_iter hti;
	struct nft_set_elem elem;
	int err;

	err = rhashtable_walk_init(&priv->ht, &hti, GFP_ATOMIC);
	iter->err = err;
	if (err)
		return;

	err = rhashtable_walk_start(&hti);
	if (err && err != -EAGAIN) {
		iter->err = err;
		goto out;
	}

	while ((he = rhashtable_walk_next(&hti))) {
		if (IS_ERR(he)) {
			err = PTR_ERR(he);
			if (err != -EAGAIN) {
				iter->err = err;
				goto out;
			}

			continue;
		}

		if (iter->count < iter->skip)
			goto cont;
		if (nft_set_elem_expired(&he->ext))
			goto cont;
		if (!nft_set_elem_active(&he->ext, iter->genmask))
			goto cont;

		elem.priv = he;

		iter->err = iter->fn(ctx, set, iter, &elem);
		if (iter->err < 0)
			goto out;

cont:
		iter->count++;
	}

out:
	rhashtable_walk_stop(&hti);
	rhashtable_walk_exit(&hti);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Patrick McHardy
131
52.19%
6
46.15%
Herbert Xu
107
42.63%
2
15.38%
Thomas Graf
6
2.39%
1
7.69%
Pablo Neira Ayuso
5
1.99%
2
15.38%
Bob Copeland
1
0.40%
1
7.69%
Liping Zhang
1
0.40%
1
7.69%
Total
251
100.00%
13
100.00%



static void nft_rhash_gc(struct work_struct *work)
{
	struct nft_set *set;
	struct nft_rhash_elem *he;
	struct nft_rhash *priv;
	struct nft_set_gc_batch *gcb = NULL;
	struct rhashtable_iter hti;
	int err;

	priv = container_of(work, struct nft_rhash, gc_work.work);
	set  = nft_set_container_of(priv);

	err = rhashtable_walk_init(&priv->ht, &hti, GFP_KERNEL);
	if (err)
		goto schedule;

	err = rhashtable_walk_start(&hti);
	if (err && err != -EAGAIN)
		goto out;

	while ((he = rhashtable_walk_next(&hti))) {
		if (IS_ERR(he)) {
			if (PTR_ERR(he) != -EAGAIN)
				goto out;
			continue;
		}

		if (!nft_set_elem_expired(&he->ext))
			continue;
		if (nft_set_elem_mark_busy(&he->ext))
			continue;

		gcb = nft_set_gc_batch_check(set, gcb, GFP_ATOMIC);
		if (gcb == NULL)
			goto out;
		rhashtable_remove_fast(&priv->ht, &he->node, nft_rhash_params);
		atomic_dec(&set->nelems);
		nft_set_gc_batch_add(gcb, he);
	}
out:
	rhashtable_walk_stop(&hti);
	rhashtable_walk_exit(&hti);

	nft_set_gc_batch_complete(gcb);
schedule:
	queue_delayed_work(system_power_efficient_wq, &priv->gc_work,
			   nft_set_gc_interval(set));
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Patrick McHardy
242
97.19%
2
50.00%
Pablo Neira Ayuso
5
2.01%
1
25.00%
Bob Copeland
2
0.80%
1
25.00%
Total
249
100.00%
4
100.00%



static unsigned int nft_rhash_privsize(const struct nlattr * const nla[],
				       const struct nft_set_desc *desc)
{
	return sizeof(struct nft_rhash);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Patrick McHardy
17
60.71%
2
40.00%
Pablo Neira Ayuso
8
28.57%
2
40.00%
Thomas Graf
3
10.71%
1
20.00%
Total
28
100.00%
5
100.00%



static int nft_rhash_init(const struct nft_set *set,
			  const struct nft_set_desc *desc,
			  const struct nlattr * const tb[])
{
	struct nft_rhash *priv = nft_set_priv(set);
	struct rhashtable_params params = nft_rhash_params;
	int err;

	params.nelem_hint = desc->size ?: NFT_RHASH_ELEMENT_HINT;
	params.key_len	  = set->klen;

	err = rhashtable_init(&priv->ht, &params);
	if (err < 0)
		return err;

	INIT_DEFERRABLE_WORK(&priv->gc_work, nft_rhash_gc);
	if (set->flags & NFT_SET_TIMEOUT)
		queue_delayed_work(system_power_efficient_wq, &priv->gc_work,
				   nft_set_gc_interval(set));
	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Patrick McHardy
96
78.69%
6
66.67%
Thomas Graf
14
11.48%
1
11.11%
Herbert Xu
7
5.74%
1
11.11%
Pablo Neira Ayuso
5
4.10%
1
11.11%
Total
122
100.00%
9
100.00%



static void nft_rhash_elem_destroy(void *ptr, void *arg)
{
	nft_set_elem_destroy(arg, ptr, true);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Thomas Graf
19
82.61%
1
25.00%
Liping Zhang
2
8.70%
1
25.00%
Patrick McHardy
1
4.35%
1
25.00%
Pablo Neira Ayuso
1
4.35%
1
25.00%
Total
23
100.00%
4
100.00%



static void nft_rhash_destroy(const struct nft_set *set)
{
	struct nft_rhash *priv = nft_set_priv(set);

	cancel_delayed_work_sync(&priv->gc_work);
	rhashtable_free_and_destroy(&priv->ht, nft_rhash_elem_destroy,
				    (void *)set);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Patrick McHardy
37
80.43%
4
50.00%
Thomas Graf
6
13.04%
3
37.50%
Pablo Neira Ayuso
3
6.52%
1
12.50%
Total
46
100.00%
8
100.00%



static u32 nft_hash_buckets(u32 size)
{
	return roundup_pow_of_two(size * 4 / 3);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Pablo Neira Ayuso
19
100.00%
1
100.00%
Total
19
100.00%
1
100.00%



static bool nft_rhash_estimate(const struct nft_set_desc *desc, u32 features,
			       struct nft_set_estimate *est)
{
	est->size   = ~0;
	est->lookup = NFT_SET_CLASS_O_1;
	est->space  = NFT_SET_CLASS_O_N;

	return true;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Pablo Neira Ayuso
24
57.14%
2
66.67%
Patrick McHardy
18
42.86%
1
33.33%
Total
42
100.00%
3
100.00%


struct nft_hash {
	
u32				seed;
	
u32				buckets;
	
struct hlist_head		table[];
};


struct nft_hash_elem {
	
struct hlist_node		node;
	
struct nft_set_ext		ext;
};



static bool nft_hash_lookup(const struct net *net, const struct nft_set *set,
			    const u32 *key, const struct nft_set_ext **ext)
{
	struct nft_hash *priv = nft_set_priv(set);
	u8 genmask = nft_genmask_cur(net);
	const struct nft_hash_elem *he;
	u32 hash;

	hash = jhash(key, set->klen, priv->seed);
	hash = reciprocal_scale(hash, priv->buckets);
	hlist_for_each_entry_rcu(he, &priv->table[hash], node) {
		if (!memcmp(nft_set_ext_key(&he->ext), key, set->klen) &&
		    nft_set_elem_active(&he->ext, genmask)) {
			*ext = &he->ext;
			return true;
		}
	}
	return false;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Pablo Neira Ayuso
139
100.00%
1
100.00%
Total
139
100.00%
1
100.00%

/* nft_hash_select_ops() makes sure key size can be either 2 or 4 bytes . */


static inline u32 nft_hash_key(const u32 *key, u32 klen)
{
	if (klen == 4)
		return *key;

	return *(u16 *)key;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Pablo Neira Ayuso
33
100.00%
1
100.00%
Total
33
100.00%
1
100.00%



static bool nft_hash_lookup_fast(const struct net *net,
				 const struct nft_set *set,
				 const u32 *key, const struct nft_set_ext **ext)
{
	struct nft_hash *priv = nft_set_priv(set);
	u8 genmask = nft_genmask_cur(net);
	const struct nft_hash_elem *he;
	u32 hash, k1, k2;

	k1 = nft_hash_key(key, set->klen);
	hash = jhash_1word(k1, priv->seed);
	hash = reciprocal_scale(hash, priv->buckets);
	hlist_for_each_entry_rcu(he, &priv->table[hash], node) {
		k2 = nft_hash_key(nft_set_ext_key(&he->ext)->data, set->klen);
		if (k1 == k2 &&
		    nft_set_elem_active(&he->ext, genmask)) {
			*ext = &he->ext;
			return true;
		}
	}
	return false;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Pablo Neira Ayuso
155
100.00%
1
100.00%
Total
155
100.00%
1
100.00%



static int nft_hash_insert(const struct net *net, const struct nft_set *set,
			   const struct nft_set_elem *elem,
			   struct nft_set_ext **ext)
{
	struct nft_hash_elem *this = elem->priv, *he;
	struct nft_hash *priv = nft_set_priv(set);
	u8 genmask = nft_genmask_next(net);
	u32 hash;

	hash = jhash(nft_set_ext_key(&this->ext), set->klen, priv->seed);
	hash = reciprocal_scale(hash, priv->buckets);
	hlist_for_each_entry(he, &priv->table[hash], node) {
		if (!memcmp(nft_set_ext_key(&this->ext),
			    nft_set_ext_key(&he->ext), set->klen) &&
		    nft_set_elem_active(&he->ext, genmask)) {
			*ext = &he->ext;
			return -EEXIST;
		}
	}
	hlist_add_head_rcu(&this->node, &priv->table[hash]);
	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Pablo Neira Ayuso
174
100.00%
1
100.00%
Total
174
100.00%
1
100.00%



static void nft_hash_activate(const struct net *net, const struct nft_set *set,
			      const struct nft_set_elem *elem)
{
	struct nft_hash_elem *he = elem->priv;

	nft_set_elem_change_active(net, set, &he->ext);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Pablo Neira Ayuso
45
100.00%
1
100.00%
Total
45
100.00%
1
100.00%



static bool nft_hash_flush(const struct net *net,
			   const struct nft_set *set, void *priv)
{
	struct nft_hash_elem *he = priv;

	nft_set_elem_change_active(net, set, &he->ext);
	return true;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Pablo Neira Ayuso
44
100.00%
1
100.00%
Total
44
100.00%
1
100.00%



static void *nft_hash_deactivate(const struct net *net,
				 const struct nft_set *set,
				 const struct nft_set_elem *elem)
{
	struct nft_hash *priv = nft_set_priv(set);
	struct nft_hash_elem *this = elem->priv, *he;
	u8 genmask = nft_genmask_next(net);
	u32 hash;

	hash = jhash(nft_set_ext_key(&this->ext), set->klen, priv->seed);
	hash = reciprocal_scale(hash, priv->buckets);
	hlist_for_each_entry(he, &priv->table[hash], node) {
		if (!memcmp(nft_set_ext_key(&this->ext), &elem->key.val,
			    set->klen) ||
		    nft_set_elem_active(&he->ext, genmask)) {
			nft_set_elem_change_active(net, set, &he->ext);
			return he;
		}
	}
	return NULL;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Pablo Neira Ayuso
155
100.00%
1
100.00%
Total
155
100.00%
1
100.00%



static void nft_hash_remove(const struct net *net,
			    const struct nft_set *set,
			    const struct nft_set_elem *elem)
{
	struct nft_hash_elem *he = elem->priv;

	hlist_del_rcu(&he->node);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Pablo Neira Ayuso
41
100.00%
1
100.00%
Total
41
100.00%
1
100.00%



static void nft_hash_walk(const struct nft_ctx *ctx, struct nft_set *set,
			  struct nft_set_iter *iter)
{
	struct nft_hash *priv = nft_set_priv(set);
	struct nft_hash_elem *he;
	struct nft_set_elem elem;
	int i;

	for (i = 0; i < priv->buckets; i++) {
		hlist_for_each_entry_rcu(he, &priv->table[i], node) {
			if (iter->count < iter->skip)
				goto cont;
			if (!nft_set_elem_active(&he->ext, iter->genmask))
				goto cont;

			elem.priv = he;

			iter->err = iter->fn(ctx, set, iter, &elem);
			if (iter->err < 0)
				return;
cont:
			iter->count++;
		}
	}
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Pablo Neira Ayuso
140
98.59%
1
50.00%
Patrick McHardy
2
1.41%
1
50.00%
Total
142
100.00%
2
100.00%



static unsigned int nft_hash_privsize(const struct nlattr * const nla[],
				      const struct nft_set_desc *desc)
{
	return sizeof(struct nft_hash) +
	       nft_hash_buckets(desc->size) * sizeof(struct hlist_head);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Pablo Neira Ayuso
41
100.00%
1
100.00%
Total
41
100.00%
1
100.00%



static int nft_hash_init(const struct nft_set *set,
			 const struct nft_set_desc *desc,
			 const struct nlattr * const tb[])
{
	struct nft_hash *priv = nft_set_priv(set);

	priv->buckets = nft_hash_buckets(desc->size)