Release 4.14 net/ipv6/esp6.c

/*
 * Copyright (C)2002 USAGI/WIDE Project
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, see <http://www.gnu.org/licenses/>.
 *
 * Authors
 *
 *      Mitsuru KANDA @USAGI       : IPv6 Support
 *      Kazunori MIYAZAWA @USAGI   :
 *      Kunihiro Ishiguro <kunihiro@ipinfusion.com>
 *
 *      This file is derived from net/ipv4/esp.c
 */


#define pr_fmt(fmt) "IPv6: " fmt

#include <crypto/aead.h>
#include <crypto/authenc.h>
#include <linux/err.h>
#include <linux/module.h>
#include <net/ip.h>
#include <net/xfrm.h>
#include <net/esp.h>
#include <linux/scatterlist.h>
#include <linux/kernel.h>
#include <linux/pfkeyv2.h>
#include <linux/random.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <net/ip6_route.h>
#include <net/icmp.h>
#include <net/ipv6.h>
#include <net/protocol.h>
#include <linux/icmpv6.h>

#include <linux/highmem.h>


struct esp_skb_cb {
	
struct xfrm_skb_cb xfrm;
	
void *tmp;
};


#define ESP_SKB_CB(__skb) ((struct esp_skb_cb *)&((__skb)->cb[0]))

static u32 esp6_get_mtu(struct xfrm_state *x, int mtu);

/*
 * Allocate an AEAD request structure with extra space for SG and IV.
 *
 * For alignment considerations the upper 32 bits of the sequence number are
 * placed at the front, if present. Followed by the IV, the request and finally
 * the SG list.
 *
 * TODO: Use spare space in skb for this where possible.
 */


static void *esp_alloc_tmp(struct crypto_aead *aead, int nfrags, int seqihlen)
{
	unsigned int len;

	len = seqihlen;

	len += crypto_aead_ivsize(aead);

	if (len) {
		len += crypto_aead_alignmask(aead) &
		       ~(crypto_tfm_ctx_alignment() - 1);
		len = ALIGN(len, crypto_tfm_ctx_alignment());
	}

	len += sizeof(struct aead_request) + crypto_aead_reqsize(aead);
	len = ALIGN(len, __alignof__(struct scatterlist));

	len += sizeof(struct scatterlist) * nfrags;

	return kmalloc(len, GFP_ATOMIC);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Herbert Xu
83
76.85%
4
66.67%
Kazunori Miyazawa
18
16.67%
1
16.67%
Steffen Klassert
7
6.48%
1
16.67%
Total
108
100.00%
6
100.00%



static inline __be32 *esp_tmp_seqhi(void *tmp)
{
	return PTR_ALIGN((__be32 *)tmp, __alignof__(__be32));
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Steffen Klassert
27
100.00%
1
100.00%
Total
27
100.00%
1
100.00%



static inline u8 *esp_tmp_iv(struct crypto_aead *aead, void *tmp, int seqhilen)
{
	return crypto_aead_ivsize(aead) ?
	       PTR_ALIGN((u8 *)tmp + seqhilen,
			 crypto_aead_alignmask(aead) + 1) : tmp + seqhilen;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Herbert Xu
39
81.25%
1
33.33%
Steffen Klassert
7
14.58%
1
33.33%
Arnaldo Carvalho de Melo
2
4.17%
1
33.33%
Total
48
100.00%
3
100.00%



static inline struct aead_request *esp_tmp_req(struct crypto_aead *aead, u8 *iv)
{
	struct aead_request *req;

	req = (void *)PTR_ALIGN(iv + crypto_aead_ivsize(aead),
				crypto_tfm_ctx_alignment());
	aead_request_set_tfm(req, aead);
	return req;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Herbert Xu
52
100.00%
1
100.00%
Total
52
100.00%
1
100.00%



static inline struct scatterlist *esp_req_sg(struct crypto_aead *aead,
					     struct aead_request *req)
{
	return (void *)ALIGN((unsigned long)(req + 1) +
			     crypto_aead_reqsize(aead),
			     __alignof__(struct scatterlist));
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Herbert Xu
48
100.00%
1
100.00%
Total
48
100.00%
1
100.00%



static void esp_ssg_unref(struct xfrm_state *x, void *tmp)
{
	struct crypto_aead *aead = x->data;
	int seqhilen = 0;
	u8 *iv;
	struct aead_request *req;
	struct scatterlist *sg;

	if (x->props.flags & XFRM_STATE_ESN)
		seqhilen += sizeof(__be32);

	iv = esp_tmp_iv(aead, tmp, seqhilen);
	req = esp_tmp_req(aead, iv);

	/* Unref skb_frag_pages in the src scatterlist if necessary.
         * Skip the first sg which comes from skb->data.
         */
	if (req->src != req->dst)
		for (sg = sg_next(req->src); sg; sg = sg_next(sg))
			put_page(sg_page(sg));
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Steffen Klassert
119
100.00%
1
100.00%
Total
119
100.00%
1
100.00%



static void esp_output_done(struct crypto_async_request *base, int err)
{
	struct sk_buff *skb = base->data;
	void *tmp;
	struct dst_entry *dst = skb_dst(skb);
	struct xfrm_state *x = dst->xfrm;

	tmp = ESP_SKB_CB(skb)->tmp;
	esp_ssg_unref(x, tmp);
	kfree(tmp);
	xfrm_output_resume(skb, err);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Herbert Xu
39
52.70%
2
66.67%
Steffen Klassert
35
47.30%
1
33.33%
Total
74
100.00%
3
100.00%

/* Move ESP header back into place. */


static void esp_restore_header(struct sk_buff *skb, unsigned int offset)
{
	struct ip_esp_hdr *esph = (void *)(skb->data + offset);
	void *tmp = ESP_SKB_CB(skb)->tmp;
	__be32 *seqhi = esp_tmp_seqhi(tmp);

	esph->seq_no = esph->spi;
	esph->spi = *seqhi;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Herbert Xu
67
100.00%
2
100.00%
Total
67
100.00%
2
100.00%



static void esp_output_restore_header(struct sk_buff *skb)
{
	esp_restore_header(skb, skb_transport_offset(skb) - sizeof(__be32));
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Herbert Xu
26
100.00%
2
100.00%
Total
26
100.00%
2
100.00%



static struct ip_esp_hdr *esp_output_set_esn(struct sk_buff *skb,
					     struct xfrm_state *x,
					     struct ip_esp_hdr *esph,
					     __be32 *seqhi)
{
	/* For ESN we move the header forward by 4 bytes to
         * accomodate the high bits.  We will move it back after
         * encryption.
         */
	if ((x->props.flags & XFRM_STATE_ESN)) {
		struct xfrm_offload *xo = xfrm_offload(skb);

		esph = (void *)(skb_transport_header(skb) - sizeof(__be32));
		*seqhi = esph->spi;
		if (xo)
			esph->seq_no = htonl(xo->seq.hi);
		else
			esph->seq_no = htonl(XFRM_SKB_CB(skb)->seq.output.hi);
	}

	esph->spi = x->id.spi;

	return esph;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Steffen Klassert
111
88.10%
3
75.00%
Herbert Xu
15
11.90%
1
25.00%
Total
126
100.00%
4
100.00%



static void esp_output_done_esn(struct crypto_async_request *base, int err)
{
	struct sk_buff *skb = base->data;

	esp_output_restore_header(skb);
	esp_output_done(base, err);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Steffen Klassert
35
100.00%
1
100.00%
Total
35
100.00%
1
100.00%



static void esp_output_fill_trailer(u8 *tail, int tfclen, int plen, __u8 proto)
{
	/* Fill padding... */
	if (tfclen) {
		memset(tail, 0, tfclen);
		tail += tfclen;
	}
	do {
		int i;
		for (i = 0; i < plen - 2; i++)
			tail[i] = i + 1;
	} while (0);
	tail[plen - 2] = plen - 2;
	tail[plen - 1] = proto;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Steffen Klassert
94
100.00%
1
100.00%
Total
94
100.00%
1
100.00%



int esp6_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info *esp)
{
	u8 *tail;
	u8 *vaddr;
	int nfrags;
	struct page *page;
	struct sk_buff *trailer;
	int tailen = esp->tailen;

	if (!skb_cloned(skb)) {
		if (tailen <= skb_tailroom(skb)) {
			nfrags = 1;
			trailer = skb;
			tail = skb_tail_pointer(trailer);

			goto skip_cow;
		} else if ((skb_shinfo(skb)->nr_frags < MAX_SKB_FRAGS)
			   && !skb_has_frag_list(skb)) {
			int allocsize;
			struct sock *sk = skb->sk;
			struct page_frag *pfrag = &x->xfrag;

			esp->inplace = false;

			allocsize = ALIGN(tailen, L1_CACHE_BYTES);

			spin_lock_bh(&x->lock);

			if (unlikely(!skb_page_frag_refill(allocsize, pfrag, GFP_ATOMIC))) {
				spin_unlock_bh(&x->lock);
				goto cow;
			}

			page = pfrag->page;
			get_page(page);

			vaddr = kmap_atomic(page);

			tail = vaddr + pfrag->offset;

			esp_output_fill_trailer(tail, esp->tfclen, esp->plen, esp->proto);

			kunmap_atomic(vaddr);

			nfrags = skb_shinfo(skb)->nr_frags;

			__skb_fill_page_desc(skb, nfrags, page, pfrag->offset,
					     tailen);
			skb_shinfo(skb)->nr_frags = ++nfrags;

			pfrag->offset = pfrag->offset + allocsize;

			spin_unlock_bh(&x->lock);

			nfrags++;

			skb->len += tailen;
			skb->data_len += tailen;
			skb->truesize += tailen;
			if (sk)
				refcount_add(tailen, &sk->sk_wmem_alloc);

			goto out;
		}
	}

cow:
	nfrags = skb_cow_data(skb, tailen, &trailer);
	if (nfrags < 0)
		goto out;
	tail = skb_tail_pointer(trailer);

skip_cow:
	esp_output_fill_trailer(tail, esp->tfclen, esp->plen, esp->proto);
	pskb_put(skb, trailer, tailen);

out:
	return nfrags;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Steffen Klassert
298
77.81%
6
50.00%
Herbert Xu
49
12.79%
3
25.00%
Martin Willi
28
7.31%
1
8.33%
Kazunori Miyazawa
7
1.83%
1
8.33%
Elena Reshetova
1
0.26%
1
8.33%
Total
383
100.00%
12
100.00%


EXPORT_SYMBOL_GPL(esp6_output_head);



int esp6_output_tail(struct xfrm_state *x, struct sk_buff *skb, struct esp_info *esp)
{
	u8 *iv;
	int alen;
	void *tmp;
	int ivlen;
	int assoclen;
	int seqhilen;
	__be32 *seqhi;
	struct page *page;
	struct ip_esp_hdr *esph;
	struct aead_request *req;
	struct crypto_aead *aead;
	struct scatterlist *sg, *dsg;
	int err = -ENOMEM;

	assoclen = sizeof(struct ip_esp_hdr);
	seqhilen = 0;

	if (x->props.flags & XFRM_STATE_ESN) {
		seqhilen += sizeof(__be32);
		assoclen += sizeof(__be32);
	}

	aead = x->data;
	alen = crypto_aead_authsize(aead);
	ivlen = crypto_aead_ivsize(aead);

	tmp = esp_alloc_tmp(aead, esp->nfrags + 2, seqhilen);
	if (!tmp)
		goto error;

	seqhi = esp_tmp_seqhi(tmp);
	iv = esp_tmp_iv(aead, tmp, seqhilen);
	req = esp_tmp_req(aead, iv);
	sg = esp_req_sg(aead, req);

	if (esp->inplace)
		dsg = sg;
	else
		dsg = &sg[esp->nfrags];

	esph = esp_output_set_esn(skb, x, ip_esp_hdr(skb), seqhi);

	sg_init_table(sg, esp->nfrags);
	err = skb_to_sgvec(skb, sg,
		           (unsigned char *)esph - skb->data,
		           assoclen + ivlen + esp->clen + alen);
	if (unlikely(err < 0))
		goto error_free;

	if (!esp->inplace) {
		int allocsize;
		struct page_frag *pfrag = &x->xfrag;

		allocsize = ALIGN(skb->data_len, L1_CACHE_BYTES);

		spin_lock_bh(&x->lock);
		if (unlikely(!skb_page_frag_refill(allocsize, pfrag, GFP_ATOMIC))) {
			spin_unlock_bh(&x->lock);
			goto error_free;
		}

		skb_shinfo(skb)->nr_frags = 1;

		page = pfrag->page;
		get_page(page);
		/* replace page frags in skb with new page */
		__skb_fill_page_desc(skb, 0, page, pfrag->offset, skb->data_len);
		pfrag->offset = pfrag->offset + allocsize;
		spin_unlock_bh(&x->lock);

		sg_init_table(dsg, skb_shinfo(skb)->nr_frags + 1);
		err = skb_to_sgvec(skb, dsg,
			           (unsigned char *)esph - skb->data,
			           assoclen + ivlen + esp->clen + alen);
		if (unlikely(err < 0))
			goto error_free;
	}

	if ((x->props.flags & XFRM_STATE_ESN))
		aead_request_set_callback(req, 0, esp_output_done_esn, skb);
	else
		aead_request_set_callback(req, 0, esp_output_done, skb);

	aead_request_set_crypt(req, sg, dsg, ivlen + esp->clen, iv);
	aead_request_set_ad(req, assoclen);

	memset(iv, 0, ivlen);
	memcpy(iv + ivlen - min(ivlen, 8), (u8 *)&esp->seqno + 8 - min(ivlen, 8),
	       min(ivlen, 8));

	ESP_SKB_CB(skb)->tmp = tmp;
	err = crypto_aead_encrypt(req);

	switch (err) {
	case -EINPROGRESS:
		goto error;

	case -EBUSY:
		err = NET_XMIT_DROP;
		break;

	case 0:
		if ((x->props.flags & XFRM_STATE_ESN))
			esp_output_restore_header(skb);
	}

	if (sg != dsg)
		esp_ssg_unref(x, tmp);

error_free:
	kfree(tmp);
error:
	return err;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Steffen Klassert
615
94.04%
5
35.71%
Jason A. Donenfeld
26
3.98%
1
7.14%
Herbert Xu
8
1.22%
4
28.57%
Arnaldo Carvalho de Melo
3
0.46%
2
14.29%
Kazunori Miyazawa
1
0.15%
1
7.14%
Jamal Hadi Salim
1
0.15%
1
7.14%
Total
654
100.00%
14
100.00%


EXPORT_SYMBOL_GPL(esp6_output_tail);



static int esp6_output(struct xfrm_state *x, struct sk_buff *skb)
{
	int alen;
	int blksize;
	struct ip_esp_hdr *esph;
	struct crypto_aead *aead;
	struct esp_info esp;

	esp.inplace = true;

	esp.proto = *skb_mac_header(skb);
	*skb_mac_header(skb) = IPPROTO_ESP;

	/* skb is pure payload to encrypt */

	aead = x->data;
	alen = crypto_aead_authsize(aead);

	esp.tfclen = 0;
	if (x->tfcpad) {
		struct xfrm_dst *dst = (struct xfrm_dst *)skb_dst(skb);
		u32 padto;

		padto = min(x->tfcpad, esp6_get_mtu(x, dst->child_mtu_cached));
		if (skb->len < padto)
			esp.tfclen = padto - skb->len;
	}
	blksize = ALIGN(crypto_aead_blocksize(aead), 4);
	esp.clen = ALIGN(skb->len + 2 + esp.tfclen, blksize);
	esp.plen = esp.clen - skb->len - esp.tfclen;
	esp.tailen = esp.tfclen + esp.plen + alen;

	esp.nfrags = esp6_output_head(x, skb, &esp);
	if (esp.nfrags < 0)
		return esp.nfrags;

	esph = ip_esp_hdr(skb);
	esph->spi = x->id.spi;

	esph->seq_no = htonl(XFRM_SKB_CB(skb)->seq.output.low);
	esp.seqno = cpu_to_be64(XFRM_SKB_CB(skb)->seq.output.low +
			    ((u64)XFRM_SKB_CB(skb)->seq.output.hi << 32));

	skb_push(skb, -skb_network_offset(skb));

	return esp6_output_tail(x, skb, &esp);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Steffen Klassert
265
82.04%
4
50.00%
Herbert Xu
57
17.65%
3
37.50%
David S. Miller
1
0.31%
1
12.50%
Total
323
100.00%
8
100.00%



static inline int esp_remove_trailer(struct sk_buff *skb)
{
	struct xfrm_state *x = xfrm_input_state(skb);
	struct xfrm_offload *xo = xfrm_offload(skb);
	struct crypto_aead *aead = x->data;
	int alen, hlen, elen;
	int padlen, trimlen;
	__wsum csumdiff;
	u8 nexthdr[2];
	int ret;

	alen = crypto_aead_authsize(aead);
	hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead);
	elen = skb->len - hlen;

	if (xo && (xo->flags & XFRM_ESP_NO_TRAILER)) {
		ret = xo->proto;
		goto out;
	}

	if (skb_copy_bits(skb, skb->len - alen - 2, nexthdr, 2))
		BUG();

	ret = -EINVAL;
	padlen = nexthdr[0];
	if (padlen + 2 + alen >= elen) {
		net_dbg_ratelimited("ipsec esp packet is garbage padlen=%d, elen=%d\n",
				    padlen + 2, elen - alen);
		goto out;
	}

	trimlen = alen + padlen + 2;
	if (skb->ip_summed == CHECKSUM_COMPLETE) {
		csumdiff = skb_checksum(skb, skb->len - trimlen, trimlen, 0);
		skb->csum = csum_block_sub(skb->csum, csumdiff,
					   skb->len - trimlen);
	}
	pskb_trim(skb, skb->len - trimlen);

	ret = nexthdr[1];

out:
	return ret;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Herbert Xu
94
36.86%
3
25.00%
Ilan Tayari
54
21.18%
1
8.33%
Yossi Kuperman
53
20.78%
1
8.33%
Kazunori Miyazawa
23
9.02%
1
8.33%
Steffen Klassert
21
8.24%
1
8.33%
David S. Miller
6
2.35%
3
25.00%
Mathias Krause
2
0.78%
1
8.33%
Joe Perches
2
0.78%
1
8.33%
Total
255
100.00%
12
100.00%



int esp6_input_done2(struct sk_buff *skb, int err)
{
	struct xfrm_state *x = xfrm_input_state(skb);
	struct xfrm_offload *xo = xfrm_offload(skb);
	struct crypto_aead *aead = x->data;
	int hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead);
	int hdr_len = skb_network_header_len(skb);

	if (!xo || (xo && !(xo->flags & CRYPTO_DONE)))
		kfree(ESP_SKB_CB(skb)->tmp);

	if (unlikely(err))
		goto out;

	err = esp_remove_trailer(skb);
	if (unlikely(err < 0))
		goto out;

	skb_postpull_rcsum(skb, skb_network_header(skb),
			   skb_network_header_len(skb));
	skb_pull_rcsum(skb, hlen);
	if (x->props.mode == XFRM_MODE_TUNNEL)
		skb_reset_transport_header(skb);
	else
		skb_set_transport_header(skb, -hdr_len);

	/* RFC4303: Drop dummy packets without any error */
	if (err == IPPROTO_NONE)
		err = -EINVAL;

out:
	return err;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Yossi Kuperman
135
73.37%
1
14.29%
Herbert Xu
25
13.59%
3
42.86%
Li RongQing
16
8.70%
1
14.29%
Kazunori Miyazawa
7
3.80%
1
14.29%
Ilan Tayari
1
0.54%
1
14.29%
Total
184
100.00%
7
100.00%


EXPORT_SYMBOL_GPL(esp6_input_done2);



static void esp_input_done(struct crypto_async_request *base, int err)
{
	struct sk_buff *skb = base->data;

	xfrm_input_resume(skb, esp6_input_done2(skb, err));
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Herbert Xu
34
97.14%
1
50.00%
Steffen Klassert
1
2.86%
1
50.00%
Total
35
100.00%
2
100.00%



static void esp_input_restore_header(struct sk_buff *skb)
{
	esp_restore_header(skb, 0);
	__skb_pull(skb, 4);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Steffen Klassert
15
60.00%
1
50.00%
Herbert Xu
10
40.00%
1
50.00%
Total
25
100.00%
2
100.00%



static void esp_input_set_header(struct sk_buff *skb, __be32 *seqhi)
{
	struct xfrm_state *x = xfrm_input_state(skb);
	struct ip_esp_hdr *esph = (struct ip_esp_hdr *)skb->data;

	/* For ESN we move the header forward by 4 bytes to
         * accomodate the high bits.  We will move it back after
         * decryption.
         */
	if ((x->props.flags & XFRM_STATE_ESN)) {
		esph = skb_push(skb, 4);
		*seqhi = esph->spi;
		esph->spi = esph->seq_no;
		esph->seq_no = XFRM_SKB_CB(skb)->seq.input.hi;
	}
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Steffen Klassert
83
89.25%
1
33.33%
Herbert Xu
10
10.75%
2
66.67%
Total
93
100.00%
3
100.00%



static void esp_input_done_esn(struct crypto_async_request *base, int err)
{
	struct sk_buff *skb = base->data;

	esp_input_restore_header(skb);
	esp_input_done(base, err);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Herbert Xu
31
88.57%
5
83.33%
Kazunori Miyazawa
4
11.43%
1
16.67%
Total
35
100.00%
6
100.00%



static int esp6_input(struct xfrm_state *x, struct sk_buff *skb)
{
	struct ip_esp_hdr *esph;
	struct crypto_aead *aead = x->data;
	struct aead_request *req;
	struct sk_buff *trailer;
	int ivlen = crypto_aead_ivsize(aead);
	int elen = skb->len - sizeof(*esph) - ivlen;
	int nfrags;
	int assoclen;
	int seqhilen;
	int ret = 0;
	void *tmp;
	__be32 *seqhi;
	u8 *iv;
	struct scatterlist *sg;

	if (!pskb_may_pull(skb, sizeof(*esph) + ivlen)) {
		ret = -EINVAL;
		goto out;
	}

	if (elen <= 0) {
		ret = -EINVAL;
		goto out;
	}

	assoclen = sizeof(